aif-ng/aif/disk/mdadm_fallback.py

import copy
import datetime
import logging
import os
import re
import subprocess
import uuid
##
import mdstat
from lxml import etree
##
import aif.disk.block_fallback as block
import aif.disk.luks_fallback as luks
import aif.disk.lvm_fallback as lvm
import aif.utils
import aif.constants


_logger = logging.getLogger(__name__)


_mdblock_size_re = re.compile(r'^(?P<sectors>[0-9]+)\s+'
                              r'\((?P<GiB>[0-9.]+)\s+GiB\s+'
                              r'(?P<GB>[0-9.]+)\s+GB\)')
_mdblock_unused_re = re.compile(r'^before=(?P<before>[0-9]+)\s+sectors,'
                                r'\s+after=(?P<after>[0-9]+)\s+sectors$')
_mdblock_badblock_re = re.compile(r'^(?P<entries>[0-9]+)\s+entries'
                                  r'[A-Za-z\s]+'
                                  r'(?P<offset>[0-9]+)\s+sectors$')


class Member(object):
    def __init__(self, member_xml, partobj):
        self.xml = member_xml
        _logger.debug('member_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
        self.device = partobj
        if not isinstance(self.device, (block.Partition,
                                        block.Disk,
                                        Array,
                                        lvm.LV,
                                        luks.LUKS)):
            _logger.error(('partobj must be of type '
                           'aif.disk.block.Partition, '
                           'aif.disk.block.Disk, or '
                           'aif.disk.mdadm.Array'))
            raise TypeError('Invalid partobj type')
        self.devpath = self.device.devpath
        self.is_superblocked = None
        self.superblock = None
        self._parseDeviceBlock()

    def _parseDeviceBlock(self):
        # I can't believe the mdstat module doesn't really have a way to do this.
        _super = subprocess.run(['mdadm', '--examine', self.devpath],
                                stdout = subprocess.PIPE,
                                stderr = subprocess.PIPE)
        _logger.info('Executed: {0}'.format(' '.join(_super.args)))
        if _super.returncode != 0:
            _logger.warning('Command returned non-zero status')
            _logger.debug('Exit status: {0}'.format(str(_super.returncode)))
            for a in ('stdout', 'stderr'):
                x = getattr(_super, a)
                if x:
                    _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip()))
            self.is_superblocked = False
            self.superblock = None
            return(None)
        block = {}
        for idx, line in enumerate(super.stdout.decode('utf-8').splitlines()):
            line = line.strip()
            if idx == 0:  # This is just the same as self.device.devpath.
                continue
            if line == '':
                continue
            k, v = [i.strip() for i in line.split(':', 1)]
            orig_k = k
            k = re.sub(r'\s+', '_', k.lower())
            if k in ('raid_devices', 'events'):
                v = int(v)
            elif k == 'magic':
                v = bytes.fromhex(v)
            elif k == 'name':
                # TODO: Will this *always* give 2 values?
                name, local_to = [i.strip() for i in v.split(None, 1)]
                local_to = re.sub(r'[()]', '', local_to)
                v = (name, local_to)
            elif k == 'raid_level':
                v = int(re.sub(r'^raid', '', v))
            elif k == 'checksum':
                cksum, status = [i.strip() for i in v.split('-')]
                v = (bytes.fromhex(cksum), status)
            elif k == 'unused_space':
                r = _mdblock_unused_re.search(v)
                if not r:
                    _logger.error('Could not parse {0} for {1}\'s superblock'.format(orig_k, self.devpath))
                    raise RuntimeError('Could not parse unused space in superblock')
                v = {}
                for i in ('before', 'after'):
                    v[i] = int(r.group(i))  # in sectors
            elif k == 'bad_block_log':
                k = 'badblock_log_entries'
                r = _mdblock_badblock_re.search(v)
                if not r:
                    _logger.error('Could not parse {0} for {1}\'s superblock'.format(orig_k, self.devpath))
                    raise RuntimeError('Could not parse badblocks in superblock')
                v = {}
                for i in ('entries', 'offset'):
                    v[i] = int(r.group(i)) # offset is in sectors
            elif k == 'array_state':
                v = [i.strip() for i in v.split(None, 1)][0].split()
            elif k == 'device_uuid':
                v = uuid.UUID(hex = v.replace(':', '-'))
            elif re.search((r'^(creation|update)_time$'), k):
                # TODO: Is this portable/correct? Or do I need to do '%a %b %d %H:%M:%s %Y'?
                v = datetime.datetime.strptime(v, '%c')
            elif re.search(r'^((avail|used)_dev|array)_size$', k):
                r = _mdblock_size_re.search(v)
                if not r:
                    _logger.error('Could not parse {0} for {1}\'s superblock'.format(orig_k, self.devpath))
                    raise RuntimeError('Could not parse size value in superblock')
                v = {}
                for i in ('sectors', 'GB', 'GiB'):
                    v[i] = float(r.group(i))
                    if i == 'sectors':
                        v[i] = int(v[i])
            elif re.search(r'^(data|super)_offset$', k):
                v = int(v.split(None, 1)[0])
            block[k] = v
        self.superblock = block
        _logger.debug('Rendered superblock info: {0}'.format(block))
        self.is_superblocked = True
        return(None)

    def prepare(self):
        if self.is_superblocked:
            cmd = subprocess.run(['mdadm', '--misc', '--zero-superblock', self.devpath],
                                 stdout = subprocess.PIPE,
                                 stderr = subprocess.PIPE)
            _logger.info('Executed: {0}'.format(' '.join(cmd.args)))
            if cmd.returncode != 0:
                _logger.warning('Command returned non-zero status')
                _logger.debug('Exit status: {0}'.format(str(cmd.returncode)))
                for a in ('stdout', 'stderr'):
                    x = getattr(cmd, a)
                    if x:
                        _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip()))
            self.is_superblocked = False
        self._parseDeviceBlock()
        return(None)


class Array(object):
    def __init__(self, array_xml, homehost, devpath = None):
        self.xml = array_xml
        _logger.debug('array_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
        self.id = self.xml.attrib['id']
        self.level = int(self.xml.attrib['level'])
        if self.level not in aif.constants.MDADM_SUPPORTED_LEVELS:
            _logger.error(('RAID level ({0}) must be one of: '
                           '{1}').format(self.level, ', '.join([str(i) for i in aif.constants.MDADM_SUPPORTED_LEVELS])))
            raise ValueError('Invalid RAID level')
        self.metadata = self.xml.attrib.get('meta', '1.2')
        if self.metadata not in aif.constants.MDADM_SUPPORTED_METADATA:
            _logger.error(('Metadata version ({0}) must be one of: '
                           '{1}').format(self.metadata, ', '.join(aif.constants.MDADM_SUPPORTED_METADATA)))
            raise ValueError('Invalid metadata version')
        self.chunksize = int(self.xml.attrib.get('chunkSize', 512))
        if self.level in (4, 5, 6, 10):
            if not aif.utils.isPowerofTwo(self.chunksize):
                # TODO: warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number?
                _logger.error('Chunksize ({0}) must be a power of 2 for RAID level {1}.'.format(self.chunksize,
                                                                                                self.level))
                raise ValueError('Invalid chunksize')
        if self.level in (0, 4, 5, 6, 10):
            if not aif.utils.hasSafeChunks(self.chunksize):
                # TODO: warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number?
                _logger.error('Chunksize ({0}) must be divisible by 4 for RAID level {1}'.format(self.chunksize,
                                                                                                 self.level))
                raise ValueError('Invalid chunksize')
        self.layout = self.xml.attrib.get('layout', 'none')
        if self.level in aif.constants.MDADM_SUPPORTED_LAYOUTS.keys():
            matcher, layout_default = aif.constants.MDADM_SUPPORTED_LAYOUTS[self.level]
            if not matcher.search(self.layout):
                if layout_default:
                    self.layout = layout_default
                else:
                    _logger.warning('Did not detect a valid layout.')
                    self.layout = None
        else:
            self.layout = None
        self.name = self.xml.attrib['name']
        self.devpath = devpath
        if not self.devpath:
            self.devpath = '/dev/md/{0}'.format(self.name)
        self.updateStatus()
        self.homehost = homehost
        self.members = []
        self.state = None
        self.info = None

    def addMember(self, memberobj):
        if not isinstance(memberobj, Member):
            _logger.error('memberobj must be of type aif.disk.mdadm.Member')
            raise TypeError('Invalid memberobj type')
        memberobj.prepare()
        self.members.append(memberobj)
        return(None)

    def create(self):
        if not self.members:
            _logger.error('Cannot create an array with no members.')
            raise RuntimeError('Missing members')
        cmd_str = ['mdadm', '--create',
                   '--name={0}'.format(self.name),
                   '--bitmap=internal',
                   '--level={0}'.format(self.level),
                   '--metadata={0}'.format(self.metadata),
                   '--chunk={0}'.format(self.chunksize),
                   '--homehost={0}'.format(self.homehost),
                   '--raid-devices={0}'.format(len(self.members))]
        if self.layout:
            cmd_str.append('--layout={0}'.format(self.layout))
        cmd_str.append(self.devpath)
        for m in self.members:
            cmd_str.append(m.devpath)
        # TODO: logging!
        cmd = subprocess.run(cmd_str, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
        _logger.info('Executed: {0}'.format(' '.join(cmd.args)))
        if cmd.returncode != 0:
            _logger.warning('Command returned non-zero status')
            _logger.debug('Exit status: {0}'.format(str(cmd.returncode)))
            for a in ('stdout', 'stderr'):
                x = getattr(cmd, a)
                if x:
                    _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip()))
            raise RuntimeError('Failed to create array successfully')
        for m in self.members:
            m._parseDeviceBlock()
        self.updateStatus()
        self.writeConf()
        self.state = 'new'
        return(None)

    def start(self, scan = False):
        _logger.info('Starting array {0}.'.format(self.name))
        if not any((self.members, self.devpath)):
            _logger.error('Cannot assemble an array with no members (for hints) or device path.')
            raise RuntimeError('Cannot start unspecified array')
        cmd_str = ['mdadm', '--assemble', self.devpath]
        if not scan:
            for m in self.members:
                cmd_str.append(m.devpath)
        else:
            cmd_str.append('--scan')
        cmd = subprocess.run(cmd_str, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
        _logger.info('Executed: {0}'.format(' '.join(cmd.args)))
        if cmd.returncode != 0:
            _logger.warning('Command returned non-zero status')
            _logger.debug('Exit status: {0}'.format(str(cmd.returncode)))
            for a in ('stdout', 'stderr'):
                x = getattr(cmd, a)
                if x:
                    _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip()))
            raise RuntimeError('Failed to start array successfully')
        self.updateStatus()
        self.state = 'assembled'
        return(None)

    def stop(self):
        _logger.error('Stopping aray {0}.'.format(self.name))
        cmd = subprocess.run(['mdadm', '--stop', self.devpath],
                             stdout = subprocess.PIPE,
                             stderr = subprocess.PIPE)
        _logger.info('Executed: {0}'.format(' '.join(cmd.args)))
        if cmd.returncode != 0:
            _logger.warning('Command returned non-zero status')
            _logger.debug('Exit status: {0}'.format(str(cmd.returncode)))
            for a in ('stdout', 'stderr'):
                x = getattr(cmd, a)
                if x:
                    _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip()))
            raise RuntimeError('Failed to stop array successfully')
        self.state = 'disassembled'
        return(None)

    def updateStatus(self):
        _info = mdstat.parse()
        for k, v in _info['devices'].items():
            if k != self.name:
                del(_info['devices'][k])
        self.info = copy.deepcopy(_info)
        _logger.debug('Rendered info: {0}'.format(_info))
        return(None)

    def writeConf(self, chroot_base):
        conf = os.path.join(chroot_base, 'etc', 'mdadm.conf')
        with open(conf, 'r') as fh:
            conflines = fh.read().splitlines()
        cmd = subprocess.run(['mdadm', '--detail', '--brief', self.devpath],
                             stdout = subprocess.PIPE,
                             stderr = subprocess.PIPE)
        _logger.info('Executed: {0}'.format(' '.join(cmd.args)))
        if cmd.returncode != 0:
            _logger.warning('Command returned non-zero status')
            _logger.debug('Exit status: {0}'.format(str(cmd.returncode)))
            for a in ('stdout', 'stderr'):
                x = getattr(cmd, a)
                if x:
                    _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip()))
            raise RuntimeError('Failed to get information about array successfully')
        arrayinfo = cmd.stdout.decode('utf-8').strip()
        if arrayinfo not in conflines:
            r = re.compile(r'^ARRAY\s+{0}'.format(self.devpath))
            nodev = True
            for l in conflines:
                if r.search(l):
                    nodev = False
                    # TODO: warning and skip instead?
                    _logger.error('An array already exists with that name but not with the same opts/GUID/etc.')
                    raise RuntimeError('Duplicate array')
            if nodev:
                with open(conf, 'a') as fh:
                    fh.write('{0}\n'.format(arrayinfo))
        return(None)