import copy import datetime import logging import os import re import subprocess import uuid ## import mdstat from lxml import etree ## import aif.disk.block_fallback as block import aif.disk.luks_fallback as luks import aif.disk.lvm_fallback as lvm import aif.utils import aif.constants _logger = logging.getLogger(__name__) _mdblock_size_re = re.compile(r'^(?P[0-9]+)\s+' r'\((?P[0-9.]+)\s+GiB\s+' r'(?P[0-9.]+)\s+GB\)') _mdblock_unused_re = re.compile(r'^before=(?P[0-9]+)\s+sectors,' r'\s+after=(?P[0-9]+)\s+sectors$') _mdblock_badblock_re = re.compile(r'^(?P[0-9]+)\s+entries' r'[A-Za-z\s]+' r'(?P[0-9]+)\s+sectors$') class Member(object): def __init__(self, member_xml, partobj): self.xml = member_xml _logger.debug('member_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) self.device = partobj if not isinstance(self.device, (block.Partition, block.Disk, Array, lvm.LV, luks.LUKS)): _logger.error(('partobj must be of type ' 'aif.disk.block.Partition, ' 'aif.disk.block.Disk, or ' 'aif.disk.mdadm.Array')) raise TypeError('Invalid partobj type') self.devpath = self.device.devpath self.is_superblocked = None self.superblock = None self._parseDeviceBlock() def _parseDeviceBlock(self): # I can't believe the mdstat module doesn't really have a way to do this. _super = subprocess.run(['mdadm', '--examine', self.devpath], stdout = subprocess.PIPE, stderr = subprocess.PIPE) _logger.info('Executed: {0}'.format(' '.join(_super.args))) if _super.returncode != 0: _logger.warning('Command returned non-zero status') _logger.debug('Exit status: {0}'.format(str(_super.returncode))) for a in ('stdout', 'stderr'): x = getattr(_super, a) if x: _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) self.is_superblocked = False self.superblock = None return(None) block = {} for idx, line in enumerate(super.stdout.decode('utf-8').splitlines()): line = line.strip() if idx == 0: # This is just the same as self.device.devpath. continue if line == '': continue k, v = [i.strip() for i in line.split(':', 1)] orig_k = k k = re.sub(r'\s+', '_', k.lower()) if k in ('raid_devices', 'events'): v = int(v) elif k == 'magic': v = bytes.fromhex(v) elif k == 'name': # TODO: Will this *always* give 2 values? name, local_to = [i.strip() for i in v.split(None, 1)] local_to = re.sub(r'[()]', '', local_to) v = (name, local_to) elif k == 'raid_level': v = int(re.sub(r'^raid', '', v)) elif k == 'checksum': cksum, status = [i.strip() for i in v.split('-')] v = (bytes.fromhex(cksum), status) elif k == 'unused_space': r = _mdblock_unused_re.search(v) if not r: _logger.error('Could not parse {0} for {1}\'s superblock'.format(orig_k, self.devpath)) raise RuntimeError('Could not parse unused space in superblock') v = {} for i in ('before', 'after'): v[i] = int(r.group(i)) # in sectors elif k == 'bad_block_log': k = 'badblock_log_entries' r = _mdblock_badblock_re.search(v) if not r: _logger.error('Could not parse {0} for {1}\'s superblock'.format(orig_k, self.devpath)) raise RuntimeError('Could not parse badblocks in superblock') v = {} for i in ('entries', 'offset'): v[i] = int(r.group(i)) # offset is in sectors elif k == 'array_state': v = [i.strip() for i in v.split(None, 1)][0].split() elif k == 'device_uuid': v = uuid.UUID(hex = v.replace(':', '-')) elif re.search((r'^(creation|update)_time$'), k): # TODO: Is this portable/correct? Or do I need to do '%a %b %d %H:%M:%s %Y'? v = datetime.datetime.strptime(v, '%c') elif re.search(r'^((avail|used)_dev|array)_size$', k): r = _mdblock_size_re.search(v) if not r: _logger.error('Could not parse {0} for {1}\'s superblock'.format(orig_k, self.devpath)) raise RuntimeError('Could not parse size value in superblock') v = {} for i in ('sectors', 'GB', 'GiB'): v[i] = float(r.group(i)) if i == 'sectors': v[i] = int(v[i]) elif re.search(r'^(data|super)_offset$', k): v = int(v.split(None, 1)[0]) block[k] = v self.superblock = block _logger.debug('Rendered superblock info: {0}'.format(block)) self.is_superblocked = True return(None) def prepare(self): if self.is_superblocked: cmd = subprocess.run(['mdadm', '--misc', '--zero-superblock', self.devpath], stdout = subprocess.PIPE, stderr = subprocess.PIPE) _logger.info('Executed: {0}'.format(' '.join(cmd.args))) if cmd.returncode != 0: _logger.warning('Command returned non-zero status') _logger.debug('Exit status: {0}'.format(str(cmd.returncode))) for a in ('stdout', 'stderr'): x = getattr(cmd, a) if x: _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) self.is_superblocked = False self._parseDeviceBlock() return(None) class Array(object): def __init__(self, array_xml, homehost, devpath = None): self.xml = array_xml _logger.debug('array_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) self.id = self.xml.attrib['id'] self.level = int(self.xml.attrib['level']) if self.level not in aif.constants.MDADM_SUPPORTED_LEVELS: _logger.error(('RAID level ({0}) must be one of: ' '{1}').format(self.level, ', '.join([str(i) for i in aif.constants.MDADM_SUPPORTED_LEVELS]))) raise ValueError('Invalid RAID level') self.metadata = self.xml.attrib.get('meta', '1.2') if self.metadata not in aif.constants.MDADM_SUPPORTED_METADATA: _logger.error(('Metadata version ({0}) must be one of: ' '{1}').format(self.metadata, ', '.join(aif.constants.MDADM_SUPPORTED_METADATA))) raise ValueError('Invalid metadata version') self.chunksize = int(self.xml.attrib.get('chunkSize', 512)) if self.level in (4, 5, 6, 10): if not aif.utils.isPowerofTwo(self.chunksize): # TODO: warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? _logger.error('Chunksize ({0}) must be a power of 2 for RAID level {1}.'.format(self.chunksize, self.level)) raise ValueError('Invalid chunksize') if self.level in (0, 4, 5, 6, 10): if not aif.utils.hasSafeChunks(self.chunksize): # TODO: warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? _logger.error('Chunksize ({0}) must be divisible by 4 for RAID level {1}'.format(self.chunksize, self.level)) raise ValueError('Invalid chunksize') self.layout = self.xml.attrib.get('layout', 'none') if self.level in aif.constants.MDADM_SUPPORTED_LAYOUTS.keys(): matcher, layout_default = aif.constants.MDADM_SUPPORTED_LAYOUTS[self.level] if not matcher.search(self.layout): if layout_default: self.layout = layout_default else: _logger.warning('Did not detect a valid layout.') self.layout = None else: self.layout = None self.name = self.xml.attrib['name'] self.devpath = devpath if not self.devpath: self.devpath = '/dev/md/{0}'.format(self.name) self.updateStatus() self.homehost = homehost self.members = [] self.state = None self.info = None def addMember(self, memberobj): if not isinstance(memberobj, Member): _logger.error('memberobj must be of type aif.disk.mdadm.Member') raise TypeError('Invalid memberobj type') memberobj.prepare() self.members.append(memberobj) return(None) def create(self): if not self.members: _logger.error('Cannot create an array with no members.') raise RuntimeError('Missing members') cmd_str = ['mdadm', '--create', '--name={0}'.format(self.name), '--bitmap=internal', '--level={0}'.format(self.level), '--metadata={0}'.format(self.metadata), '--chunk={0}'.format(self.chunksize), '--homehost={0}'.format(self.homehost), '--raid-devices={0}'.format(len(self.members))] if self.layout: cmd_str.append('--layout={0}'.format(self.layout)) cmd_str.append(self.devpath) for m in self.members: cmd_str.append(m.devpath) # TODO: logging! cmd = subprocess.run(cmd_str, stdout = subprocess.PIPE, stderr = subprocess.PIPE) _logger.info('Executed: {0}'.format(' '.join(cmd.args))) if cmd.returncode != 0: _logger.warning('Command returned non-zero status') _logger.debug('Exit status: {0}'.format(str(cmd.returncode))) for a in ('stdout', 'stderr'): x = getattr(cmd, a) if x: _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) raise RuntimeError('Failed to create array successfully') for m in self.members: m._parseDeviceBlock() self.updateStatus() self.writeConf() self.state = 'new' return(None) def start(self, scan = False): _logger.info('Starting array {0}.'.format(self.name)) if not any((self.members, self.devpath)): _logger.error('Cannot assemble an array with no members (for hints) or device path.') raise RuntimeError('Cannot start unspecified array') cmd_str = ['mdadm', '--assemble', self.devpath] if not scan: for m in self.members: cmd_str.append(m.devpath) else: cmd_str.append('--scan') cmd = subprocess.run(cmd_str, stdout = subprocess.PIPE, stderr = subprocess.PIPE) _logger.info('Executed: {0}'.format(' '.join(cmd.args))) if cmd.returncode != 0: _logger.warning('Command returned non-zero status') _logger.debug('Exit status: {0}'.format(str(cmd.returncode))) for a in ('stdout', 'stderr'): x = getattr(cmd, a) if x: _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) raise RuntimeError('Failed to start array successfully') self.updateStatus() self.state = 'assembled' return(None) def stop(self): _logger.error('Stopping aray {0}.'.format(self.name)) cmd = subprocess.run(['mdadm', '--stop', self.devpath], stdout = subprocess.PIPE, stderr = subprocess.PIPE) _logger.info('Executed: {0}'.format(' '.join(cmd.args))) if cmd.returncode != 0: _logger.warning('Command returned non-zero status') _logger.debug('Exit status: {0}'.format(str(cmd.returncode))) for a in ('stdout', 'stderr'): x = getattr(cmd, a) if x: _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) raise RuntimeError('Failed to stop array successfully') self.state = 'disassembled' return(None) def updateStatus(self): _info = mdstat.parse() for k, v in _info['devices'].items(): if k != self.name: del(_info['devices'][k]) self.info = copy.deepcopy(_info) _logger.debug('Rendered info: {0}'.format(_info)) return(None) def writeConf(self, chroot_base): conf = os.path.join(chroot_base, 'etc', 'mdadm.conf') with open(conf, 'r') as fh: conflines = fh.read().splitlines() cmd = subprocess.run(['mdadm', '--detail', '--brief', self.devpath], stdout = subprocess.PIPE, stderr = subprocess.PIPE) _logger.info('Executed: {0}'.format(' '.join(cmd.args))) if cmd.returncode != 0: _logger.warning('Command returned non-zero status') _logger.debug('Exit status: {0}'.format(str(cmd.returncode))) for a in ('stdout', 'stderr'): x = getattr(cmd, a) if x: _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) raise RuntimeError('Failed to get information about array successfully') arrayinfo = cmd.stdout.decode('utf-8').strip() if arrayinfo not in conflines: r = re.compile(r'^ARRAY\s+{0}'.format(self.devpath)) nodev = True for l in conflines: if r.search(l): nodev = False # TODO: warning and skip instead? _logger.error('An array already exists with that name but not with the same opts/GUID/etc.') raise RuntimeError('Duplicate array') if nodev: with open(conf, 'a') as fh: fh.write('{0}\n'.format(arrayinfo)) return(None)