From f4249389136aef56cb3bb0722b343742fdd19b3c Mon Sep 17 00:00:00 2001 From: brent s Date: Wed, 6 Nov 2019 07:33:15 -0500 Subject: [PATCH] mdadm done --- aif/disk/luks.py | 6 +- aif/disk/luks_fallback.py | 3 +- aif/disk/lvm.py | 8 +- aif/disk/mdadm.py | 173 ++++++++++++++++++++++++++++++++----- aif/disk/mdadm_fallback.py | 13 ++- 5 files changed, 177 insertions(+), 26 deletions(-) diff --git a/aif/disk/luks.py b/aif/disk/luks.py index 784b815..702b51b 100644 --- a/aif/disk/luks.py +++ b/aif/disk/luks.py @@ -4,10 +4,12 @@ import aif.disk.lvm as lvm import aif.disk.mdadm as mdadm -BlockDev = _common.BlockDev +_BlockDev = _common.BlockDev class LUKS(object): - def __init__(self, partobj): + def __init__(self, luks_xml, partobj): + self.xml = luks_xml + _common.addBDPlugin('crypto') self.devpath = None pass diff --git a/aif/disk/luks_fallback.py b/aif/disk/luks_fallback.py index 5e7a526..e0bcf89 100644 --- a/aif/disk/luks_fallback.py +++ b/aif/disk/luks_fallback.py @@ -4,6 +4,7 @@ import aif.disk.mdadm_fallback as mdadm class LUKS(object): - def __init__(self, partobj): + def __init__(self, luks_xml, partobj): + self.xml = luks_xml self.devpath = None pass diff --git a/aif/disk/lvm.py b/aif/disk/lvm.py index 06b3114..317f6c9 100644 --- a/aif/disk/lvm.py +++ b/aif/disk/lvm.py @@ -8,17 +8,23 @@ _BlockDev = _common.BlockDev class PV(object): - def __init__(self, partobj): + def __init__(self, pv_xml, partobj): + self.xml = pv_xml + _common.addBDPlugin('lvm') self.devpath = None pass class VG(object): def __init__(self, vg_xml, lv_objs): + self.xml = vg_xml + _common.addBDPlugin('lvm') self.devpath = None pass class LV(object): def __init__(self, lv_xml, pv_objs): + self.xml = lv_xml + _common.addBDPlugin('lvm') pass diff --git a/aif/disk/mdadm.py b/aif/disk/mdadm.py index babf511..2cf8e8c 100644 --- a/aif/disk/mdadm.py +++ b/aif/disk/mdadm.py @@ -1,4 +1,6 @@ +import datetime import re +import uuid ## import aif.utils import aif.constants @@ -11,16 +13,6 @@ import aif.disk.lvm as lvm _BlockDev = _common.BlockDev -_mdblock_size_re = re.compile(r'^(?P[0-9]+)\s+' - r'\((?P[0-9.]+)\s+GiB\s+' - r'(?P[0-9.]+)\s+GB\)') -_mdblock_unused_re = re.compile(r'^before=(?P[0-9]+)\s+sectors,' - r'\s+after=(?P[0-9]+)\s+sectors$') -_mdblock_badblock_re = re.compile(r'^(?P[0-9]+)\s+entries' - r'[A-Za-z\s]+' - r'(?P[0-9]+)\s+sectors$') - - class Member(object): def __init__(self, member_xml, partobj): self.xml = member_xml @@ -36,16 +28,46 @@ class Member(object): 'aif.disk.luks.LUKS, ' 'aif.disk.lvm.LV, or' 'aif.disk.mdadm.Array')) + _common.addBDPlugin('mdraid') self.devpath = self.device.devpath self.is_superblocked = None self.superblock = None self._parseDeviceBlock() def _parseDeviceBlock(self): - pass + # TODO: parity with mdadm_fallback.Member._parseDeviceBlock + # key names currently (probably) don't match and need to confirm the information's all present + block = {} + try: + _block = _BlockDev.md.examine(self.devpath) + except _BlockDev.MDRaidError: + self.is_superblocked = False + self.superblock = None + return() + for k in dir(_block): + if k.startswith('_'): + continue + elif k in ('copy', 'eval'): + continue + v = getattr(_block, k) + if k == 'level': + v = int(re.sub(r'^raid', '', v)) + elif k == 'update_time': + v = datetime.datetime.fromtimestamp(v) + elif re.search('^(dev_)?uuid$', k): + v = uuid.UUID(hex = v) + block[k] = v + self.superblock = block + self.is_superblocked = True + return() def prepare(self): - pass + try: + _BlockDev.md.denominate(self.devpath) + except _BlockDev.MDRaidError: + pass + _BlockDev.md.destroy(self.devpath) + return() class Array(object): @@ -60,22 +82,133 @@ class Array(object): self.metadata = self.xml.attrib.get('meta', '1.2') if self.metadata not in aif.constants.MDADM_SUPPORTED_METADATA: raise ValueError('Metadata version must be one of: {0}'.format(', '.join( - aif.constants.MDADM_SUPPORTED_METADATA))) + aif.constants.MDADM_SUPPORTED_METADATA))) + _common.addBDPlugin('mdraid') + self.chunksize = int(self.xml.attrib.get('chunkSize', 512)) + if self.level in (4, 5, 6, 10): + if not aif.utils.isPowerofTwo(self.chunksize): + # TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? + raise ValueError('chunksize must be a power of 2 for the RAID level you specified') + if self.level in (0, 4, 5, 6, 10): + if not aif.utils.hasSafeChunks(self.chunksize): + # TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? + raise ValueError('chunksize must be divisible by 4 for the RAID level you specified') + self.layout = self.xml.attrib.get('layout', 'none') + if self.level in aif.constants.MDADM_SUPPORTED_LAYOUTS.keys(): + matcher, layout_default = aif.constants.MDADM_SUPPORTED_LAYOUTS[self.level] + if not matcher.search(self.layout): + if layout_default: + self.layout = layout_default + else: + self.layout = None # TODO: log.warn? + else: + self.layout = None + self.devname = self.xml.attrib['name'] + self.fulldevname = '{0}:{1}'.format(self.homehost, self.devname) + self.devpath = devpath + if not self.devpath: + self.devpath = '/dev/md/{0}'.format(self.devname) + self.updateStatus() + self.homehost = homehost + self.members = [] + self.state = None + self.info = None def addMember(self, memberobj): - pass + if not isinstance(memberobj, Member): + raise ValueError('memberobj must be of type aif.disk.mdadm.Member') + memberobj.prepare() + self.members.append(memberobj) + return() def create(self): - pass + if not self.members: + raise RuntimeError('Cannot create an array with no members') + opts = [_BlockDev.ExtraArg.new('--homehost', + self.homehost), + _BlockDev.ExtraArg.new('--name', + self.devname)] + if self.layout: + opts.append(_BlockDev.ExtraArg.new('--layout', + self.layout)) + _BlockDev.md.create(self.devname, + str(self.level), + [i.devpath for i in self.members], + 0, + self.metadata, + True, + (self.chunksize * 1024), + opts) + for m in self.members: + m._parseDeviceBlock() + self.updateStatus() + self.writeConf() + self.devpath = self.info['device'] + self.state = 'new' + return() def start(self, scan = False): - pass + if not any((self.members, self.devpath)): + raise RuntimeError('Cannot assemble an array with no members (for hints) or device path') + if scan: + target = None + else: + target = self.devname + _BlockDev.md.activate(target, + [i.devpath for i in self.members], # Ignored if scan mode enabled + None, + True, + None) + self.state = 'assembled' + return() def stop(self): - pass + _BlockDev.md.deactivate(self.devname) + self.state = 'disassembled' + return() def updateStatus(self): - pass + _status = _BlockDev.md.detail(self.devname) + # TODO: parity with mdadm_fallback.Array.updateStatus + # key names currently (probably) don't match and need to confirm the information's all present + info = {} + for k in dir(_status): + if k.startswith('_'): + continue + elif k in ('copy',): + continue + v = getattr(_status, k) + if k == 'level': + v = int(re.sub(r'^raid', '', v)) + elif k == 'creation_time': + # TODO: Is this portable/correct? Or do I need to do something like '%a %b %d %H:%M:%s %Y'? + v = datetime.datetime.strptime(v, '%c') + elif k == 'uuid': + v = uuid.UUID(hex = v) + info[k] = v + self.info = info + return() - def writeConf(self): - pass + def writeConf(self, conf = '/etc/mdadm.conf'): + with open(conf, 'r') as fh: + conflines = fh.read().splitlines() + arrayinfo = ('ARRAY ' + '{device} ' + 'metadata={metadata} ' + 'name={name} ' + 'UUID={converted_uuid}').format(**self.info, + converted_uuid = _BlockDev.md.get_md_uuid(str(self.info['uuid']))) + if arrayinfo not in conflines: + r = re.compile(r'^ARRAY\s+{0}'.format(self.info['device'])) + nodev = True + for l in conflines: + if r.search(l): + nodev = False + # TODO: logging? + # and/or Raise an exception here; + # an array already exists with that name but not with the same opts/GUID/etc. + break + if nodev: + with open(conf, 'a') as fh: + fh.write('{0}\n'.format(arrayinfo)) + return() diff --git a/aif/disk/mdadm_fallback.py b/aif/disk/mdadm_fallback.py index 8bdedf5..315f83f 100644 --- a/aif/disk/mdadm_fallback.py +++ b/aif/disk/mdadm_fallback.py @@ -48,7 +48,8 @@ class Member(object): if super.returncode != 0: # TODO: logging? self.is_superblocked = False - return(None) + self.superblock = None + return() block = {} for idx, line in enumerate(super.stdout.decode('utf-8').splitlines()): line = line.strip() @@ -69,7 +70,7 @@ class Member(object): local_to = re.sub(r'[()]', '', local_to) v = (name, local_to) elif k == 'raid_level': - v = re.sub(r'^raid', '', v) + v = int(re.sub(r'^raid', '', v)) elif k == 'checksum': cksum, status = [i.strip() for i in v.split('-')] v = (bytes.fromhex(cksum), status) @@ -158,6 +159,8 @@ class Array(object): self.layout = None self.devname = self.xml.attrib['name'] self.devpath = devpath + if not self.devpath: + self.devpath = '/dev/md/{0}'.format(self.devname) self.updateStatus() self.homehost = homehost self.members = [] @@ -175,6 +178,8 @@ class Array(object): if not self.members: raise RuntimeError('Cannot create an array with no members') cmd = ['mdadm', '--create', + '--name={0}'.format(self.devname), + '--bitmap=internal', '--level={0}'.format(self.level), '--metadata={0}'.format(self.metadata), '--chunk={0}'.format(self.chunksize), @@ -187,6 +192,9 @@ class Array(object): cmd.append(m.devpath) # TODO: logging! subprocess.run(cmd) + for m in self.members: + m._parseDeviceBlock() + self.updateStatus() self.writeConf() self.state = 'new' return() @@ -202,6 +210,7 @@ class Array(object): cmd.append('--scan') # TODO: logging! subprocess.run(cmd) + self.updateStatus() self.state = 'assembled' return()