added support for partition flags in config, need to add in-code

This commit is contained in:
brent s 2019-10-29 01:01:31 -04:00
parent 036dd24098
commit 3a6e8843fe
4 changed files with 201 additions and 16 deletions

47
aif.xsd
View File

@ -75,6 +75,31 @@
</xs:restriction> </xs:restriction>
</xs:simpleType> </xs:simpleType>


<xs:simpleType name="t_part_flags">
<!-- parted.partition.partitionFlag -->
<xs:restriction base="xs:string">
<xs:enumeration value="atvrecv"/>
<xs:enumeration value="bios_grub"/>
<xs:enumeration value="boot"/>
<xs:enumeration value="diag"/>
<xs:enumeration value="esp"/>
<xs:enumeration value="hidden"/>
<xs:enumeration value="hp-service"/>
<xs:enumeration value="irst"/>
<xs:enumeration value="lba"/>
<xs:enumeration value="legacy_boot"/>
<xs:enumeration value="lvm"/>
<xs:enumeration value="msftdata"/>
<xs:enumeration value="msftres"/>
<xs:enumeration value="palo"/>
<xs:enumeration value="prep"/>
<xs:enumeration value="raid"/>
<xs:enumeration value="root"/>
<xs:enumeration value="swap"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_iface"> <xs:simpleType name="t_iface">
<xs:restriction base="xs:token"> <xs:restriction base="xs:token">
<!-- https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c. <!-- https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c.
@ -224,6 +249,16 @@
</xs:restriction> </xs:restriction>
</xs:simpleType> </xs:simpleType>


<xs:simpleType name="t_raid_layout">
<!-- mdadm(8), "layout=" option -->
<!-- We don't need to cook in the "faulty" levels. -->
<xs:restriction base="xs:token">
<xs:pattern
value="((left|right)-a?symmetric(-6)?|[lr][as]|parity-(fir|la)st|ddf-(zero|N)-restart|ddf-N-continue|parity-first-6|[nof][0-9]+|none)"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_nonempty"> <xs:simpleType name="t_nonempty">
<xs:restriction base="xs:token"> <xs:restriction base="xs:token">
<xs:minLength value="1"/> <xs:minLength value="1"/>
@ -283,6 +318,11 @@
<xs:sequence> <xs:sequence>
<xs:element name="part" minOccurs="1" maxOccurs="unbounded"> <xs:element name="part" minOccurs="1" maxOccurs="unbounded">
<xs:complexType> <xs:complexType>
<xs:sequence minOccurs="0" maxOccurs="1">
<xs:element name="partitionFlag" minOccurs="1"
maxOccurs="unbounded"
type="aif:t_part_flags"/>
</xs:sequence>
<xs:attribute name="id" type="aif:t_nonempty" <xs:attribute name="id" type="aif:t_nonempty"
use="required"/> use="required"/>
<xs:attribute name="name" type="aif:t_nonempty" <xs:attribute name="name" type="aif:t_nonempty"
@ -422,6 +462,13 @@
<xs:attribute name="meta" use="optional" default="1.2" <xs:attribute name="meta" use="optional" default="1.2"
type="aif:t_raid_meta"/> type="aif:t_raid_meta"/>
<xs:attribute name="level" use="required" type="aif:t_raid_levels"/> <xs:attribute name="level" use="required" type="aif:t_raid_levels"/>
<!-- KB *only*. -->
<!-- Can be pretty important!
https://www.zdnet.com/article/chunks-the-hidden-key-to-raid-performance/ -->
<xs:attribute name="chunkSize" use="optional" type="xs:positiveInteger"
default="512"/>
<xs:attribute name="layout" use="optional" type="aif:t_raid_layout"
default="none"/>
</xs:complexType> </xs:complexType>
</xs:element> </xs:element>
</xs:sequence> </xs:sequence>

View File

@ -1,8 +1,9 @@
import copy
import os import os
import re import re
## ##
import requests import requests
from lxml import etree from lxml import etree, objectify


_patterns = {'raw': re.compile(r'^\s*(?P<xml><(\?xml|aif)\s+.*)\s*$', re.DOTALL|re.MULTILINE), _patterns = {'raw': re.compile(r'^\s*(?P<xml><(\?xml|aif)\s+.*)\s*$', re.DOTALL|re.MULTILINE),
'remote': re.compile(r'^(?P<uri>(?P<proto>(https?|ftps?)://)(?P<path>.*))\s*$'), 'remote': re.compile(r'^(?P<uri>(?P<proto>(https?|ftps?)://)(?P<path>.*))\s*$'),
@ -19,6 +20,7 @@ class Config(object):
self.raw = None self.raw = None
self.xsd = None self.xsd = None
self.defaultsParser = None self.defaultsParser = None
self.obj = None


def main(self, validate = True, populate_defaults = True): def main(self, validate = True, populate_defaults = True):
self.fetch() self.fetch()
@ -27,6 +29,7 @@ class Config(object):
self.populateDefaults() self.populateDefaults()
if validate: if validate:
self.validate() self.validate()
self.pythonize()
return() return()


def fetch(self): # Just a fail-safe; this is overridden by specific subclasses. def fetch(self): # Just a fail-safe; this is overridden by specific subclasses.
@ -53,7 +56,7 @@ class Config(object):
if len(split_url) == 2: # a properly defined schemaLocation if len(split_url) == 2: # a properly defined schemaLocation
schemaURL = split_url[1] schemaURL = split_url[1]
else: else:
schemaURL = split_url[0] schemaURL = split_url[0] # a LAZY schemaLocation
req = requests.get(schemaURL) req = requests.get(schemaURL)
if not req.ok: if not req.ok:
# TODO: logging! # TODO: logging!
@ -82,21 +85,63 @@ class Config(object):
self.parseRaw(parser = self.defaultsParser) self.parseRaw(parser = self.defaultsParser)
return() return()


def pythonize(self, stripped = True, obj = 'tree'):
# https://bugs.launchpad.net/lxml/+bug/1850221
strobj = self.toString(stripped = stripped, obj = obj)
self.obj = objectify.fromstring(strobj)
objectify.annotate(self.obj)
objectify.xsiannotate(self.obj)
return()

def removeDefaults(self): def removeDefaults(self):
self.parseRaw() self.parseRaw()
return() return()


def stripNS(self): def stripNS(self, obj = None):
# https://stackoverflow.com/questions/30232031/how-can-i-strip-namespaces-out-of-an-lxml-tree/30233635#30233635 # https://stackoverflow.com/questions/30232031/how-can-i-strip-namespaces-out-of-an-lxml-tree/30233635#30233635
for x in (self.tree, self.xml): xpathq = "descendant-or-self::*[namespace-uri()!='']"
for e in x.xpath("descendant-or-self::*[namespace-uri()!='']"): if not obj:
for x in (self.tree, self.xml):
for e in x.xpath(xpathq):
e.tag = etree.QName(e).localname
elif isinstance(obj, (etree._Element, etree._ElementTree)):
obj = copy.deepcopy(obj)
for e in obj.xpath(xpathq):
e.tag = etree.QName(e).localname e.tag = etree.QName(e).localname
return(obj)
else:
raise ValueError('Did not know how to parse obj parameter')
return() return()


def toString(self, stripped = False, obj = None):
if isinstance(obj, (etree._Element, etree._ElementTree)):
if stripped:
obj = self.stripNS(obj)
elif obj in ('tree', None):
if not stripped:
obj = self.namespaced_tree
else:
obj = self.tree
elif obj == 'xml':
if not stripped:
obj = self.namespaced_xml
else:
obj = self.xml
else:
raise ValueError(('obj parameter must be "tree", "xml", or of type '
'lxml.etree._Element or lxml.etree._ElementTree'))
obj = copy.deepcopy(obj)
strxml = etree.tostring(obj,
encoding = 'utf-8',
xml_declaration = True,
pretty_print = True,
with_tail = True,
inclusive_ns_prefixes = True)
return(strxml)

def validate(self): def validate(self):
if not self.xsd: if not self.xsd:
self.getXSD() self.getXSD()
self.xsd.assertValid(self.tree)
self.xsd.assertValid(self.namespaced_tree) self.xsd.assertValid(self.namespaced_tree)
return() return()



View File

@ -1,4 +1,6 @@
import copy import copy
import math
import re
import subprocess import subprocess
## ##
import mdstat import mdstat
@ -8,6 +10,30 @@ from aif.disk.block import Partition




SUPPORTED_LEVELS = (0, 1, 4, 5, 6) SUPPORTED_LEVELS = (0, 1, 4, 5, 6)
SUPPORTED_METADATA = ('0', '0.90', '1', '1.0', '1.1', '1.2', 'default', 'ddf', 'imsm')
SUPPORTED_LAYOUTS = {5: (re.compile(r'^((left|right)-a?symmetric|[lr][as]|'
r'parity-(fir|la)st|'
r'ddf-(N|zero)-restart|ddf-N-continue)$'),
'left-symmetric'),
6: (re.compile(r'^((left|right)-a?symmetric(-6)?|[lr][as]|'
r'parity-(fir|la)st|'
r'ddf-(N|zero)-restart|ddf-N-continue|'
r'parity-first-6)$'),
None),
10: (re.compile(r'^[nof][0-9]+$'),
None)}


def _itTakesTwo(n):
# So dumb.
isPowerOf2 = math.ceil(math.log(n, 2)) == math.floor(math.log(n, 2))
return(isPowerOf2)

def _safeChunks(n):
if (n % 4) != 0:
return(False)
return(True)



class Member(object): class Member(object):
def __init__(self, member_xml, partobj): def __init__(self, member_xml, partobj):
@ -24,27 +50,79 @@ class Member(object):
return() return()


class Array(object): class Array(object):
def __init__(self, array_xml): def __init__(self, array_xml, homehost):
self.xml = array_xml self.xml = array_xml
self.id = array_xml.attrib['id'] self.id = array_xml.attrib['id']
self.level = int(array_xml.attrib['level']) self.level = int(self.xml.attrib['level'])
if self.level not in SUPPORTED_LEVELS: if self.level not in SUPPORTED_LEVELS:
raise ValueError('RAID level must be one of: {0}'.format(', '.join(SUPPORTED_LEVELS))) raise ValueError('RAID level must be one of: {0}'.format(', '.join([str(i) for i in SUPPORTED_LEVELS])))
self.metadata = self.xml.attrib.get('meta', '1.2')
if self.metadata not in SUPPORTED_METADATA:
raise ValueError('Metadata version must be one of: {0}'.format(', '.join(SUPPORTED_METADATA)))
self.chunksize = int(self.xml.attrib.get('chunkSize', 512))
if self.level in (4, 5, 6, 10):
if not _itTakesTwo(self.chunksize):
# TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number?
raise ValueError('chunksize must be a power of 2 for the RAID level you specified')
if self.level in (0, 4, 5, 6, 10):
if not _safeChunks(self.chunksize):
# TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number?
raise ValueError('chunksize must be divisible by 4 for the RAID level you specified')
self.layout = self.xml.attrib.get('layout', 'none')
if self.level in SUPPORTED_LAYOUTS.keys():
matcher, layout_default = SUPPORTED_LAYOUTS[self.level]
if not matcher.search(self.layout):
if layout_default:
self.layout = layout_default
else:
self.layout = None # TODO: log.warn?
else:
self.layout = None
self.devname = self.xml.attrib['name'] self.devname = self.xml.attrib['name']
self.devpath = '/dev/md/{0}'.format(self.devname) self.devpath = '/dev/md/{0}'.format(self.devname)
self.updateStatus() self.updateStatus()
self.members = [] self.members = []
self.state = None


def addMember(self, memberobj): def addMember(self, memberobj):
if not isinstance(memberobj, Member): if not isinstance(memberobj, Member):
raise ValueError('memberobj must be of type aif.disk.mdadm.Member') raise ValueError('memberobj must be of type aif.disk.mdadm.Member')


def assemble(self): pass
return()

def assemble(self, scan = False):
cmd = ['mdadm', '--assemble', self.devpath] cmd = ['mdadm', '--assemble', self.devpath]
if not scan:
for m in self.members:
cmd.append(m.devpath)
else:
cmd.extend([''])
# TODO: logging!
subprocess.run(cmd)

pass
return()

def create(self):
if not self.members:
raise RuntimeError('Cannot create an array with no members')
cmd = ['mdadm', '--create',
'--level={0}'.format(self.level),
'--metadata={0}'.format(self.metadata),
'--chunk={0}'.format(self.chunksize),
'--raid-devices={0}'.format(len(self.members))]
if self.layout:
cmd.append('--layout={0}'.format(self.layout))
cmd.append(self.devpath)
for m in self.members: for m in self.members:
cmd.append(m.devpath) cmd.append(m.devpath)
# TODO: logging!
subprocess.run(cmd) subprocess.run(cmd)


pass
return()

def stop(self): def stop(self):
# TODO: logging # TODO: logging
subprocess.run(['mdadm', '--stop', self.devpath]) subprocess.run(['mdadm', '--stop', self.devpath])
@ -57,3 +135,6 @@ class Array(object):
del(_info['devices'][k]) del(_info['devices'][k])
self.info = copy.deepcopy(_info) self.info = copy.deepcopy(_info)
return() return()

def writeConf(self, conf = '/etc/mdadm.conf'):
pass

View File

@ -8,12 +8,24 @@
<disk device="/dev/sda" diskFormat="gpt"> <disk device="/dev/sda" diskFormat="gpt">
<!-- Partitions are numbered *in the order they are specified*. --> <!-- Partitions are numbered *in the order they are specified*. -->
<!-- e.g. "boot" would be /dev/sda1, "secrets1" would be /dev/sda2, etc. --> <!-- e.g. "boot" would be /dev/sda1, "secrets1" would be /dev/sda2, etc. -->
<part id="boot" name="BOOT" label="/boot" start="0%" stop="10%" fsType="fat32"/> <part id="boot" name="BOOT" label="/boot" start="0%" stop="10%" fsType="fat32">
<part id="secrets1" name="crypted" label="shh" start="10%" stop="20%" fsType="ext4"/> <partitionFlag>esp</partitionFlag>
<part id="lvm_member1" name="jbod" label="dynamic" start="20%" stop="30%" fsType="ext4"/> </part>
<part id="raid1_d1" start="30%" stop="55%" fsType="ext4"/> <part id="secrets1" name="crypted" label="shh" start="10%" stop="20%" fsType="ext4">
<part id="raid1_d2" start="55%" stop="80%" fsType="ext4"/> <partitionFlag>root</partitionFlag>
<part id="swap" start="80%" stop="100%" fsType="linux-swap(v1)"/> </part>
<part id="lvm_member1" name="jbod" label="dynamic" start="20%" stop="30%" fsType="ext4">
<partitionFlag>lvm</partitionFlag>
</part>
<part id="raid1_d1" start="30%" stop="55%" fsType="ext4">
<partitionFlag>raid</partitionFlag>
</part>
<part id="raid1_d2" start="55%" stop="80%" fsType="ext4">
<partitionFlag>raid</partitionFlag>
</part>
<part id="swap" start="80%" stop="100%" fsType="linux-swap(v1)">
<partitionFlag>swap</partitionFlag>
</part>
</disk> </disk>
</blockDevices> </blockDevices>
<!-- "Special" devices are processed *in the order they are specified*. This is important if you wish to <!-- "Special" devices are processed *in the order they are specified*. This is important if you wish to