From 27978786b8179b074970abe361aa559e5984d352 Mon Sep 17 00:00:00 2001 From: brent s Date: Mon, 4 Nov 2019 23:18:28 -0500 Subject: [PATCH] making a LOT of headway on the gi stuff but hitting a namespace issue --- aif/__init__.py | 9 ++- aif/constants.py | 35 ++++++---- aif/constants_fallback.py | 21 ++++++ aif/disk/__init__.py | 5 ++ aif/disk/_common.py | 12 ++-- aif/disk/block.py | 133 ++++++++++++++++++++++++++++++++++++- aif/disk/block_fallback.py | 63 ++++++------------ aif/utils.py | 33 ++++++++- 8 files changed, 247 insertions(+), 64 deletions(-) create mode 100644 aif/constants_fallback.py diff --git a/aif/__init__.py b/aif/__init__.py index 9de48b4..7e99210 100644 --- a/aif/__init__.py +++ b/aif/__init__.py @@ -1,13 +1,20 @@ +try: + from . import constants +except ImportError: + from . import constants_fallback as constants + from . import disk from . import system from . import config -from . import constants from . import envsetup from . import log from . import network from . import pacman from . import utils + + + class AIF(object): def __init__(self): pass diff --git a/aif/constants.py b/aif/constants.py index 084c045..2eea551 100644 --- a/aif/constants.py +++ b/aif/constants.py @@ -1,13 +1,22 @@ -arch_releng_key = '4AA4767BBC9C4B1D18AE28B77F2D434B9741E8AC' -version = '0.2.0' -external_deps = ['blkinfo', - 'gpg', - 'lxml', - 'mdstat', - 'passlib', - 'psutil', - 'pyparted', - 'pyroute2', - 'pytz', - 'requests', - 'validators'] \ No newline at end of file +from .constants_fallback import * +## +import aif.disk._common +_BlockDev = aif.disk._common.BlockDev +aif.disk._common.addBDPlugin('part') + + +# LIBBLOCKDEV FLAG INDEXING / PARTED <=> LIBBLOCKDEV FLAG CONVERSION +BD_PART_FLAGS = _BlockDev.PartFlag(-1) +BD_PART_FLAGS_FRIENDLY = dict(zip(BD_PART_FLAGS.value_nicks, BD_PART_FLAGS.value_names)) +BD_PARTED_MAP = {'apple_tv_recovery': 'atvrecv', + 'cpalo': 'palo', + 'gpt_hidden': None, # ??? + 'gpt_no_automount': None, # ??? + 'gpt_read_only': None, # ??? + 'gpt_system_part': None, # ??? + 'hpservice': 'hp-service', + 'msft_data': 'msftdata', + 'msft_reserved': 'msftres'} +PARTED_BD_MAP = {v: k for k, v in BD_PARTED_MAP.items() if v is not None} +BD_PART_FLAGS_IDX_FLAG = {k: v.value_nicks[0] for k, v in BD_PART_FLAGS.__flags_values__.items()} +BD_PART_FLAGS_FLAG_IDX = {v: k for k, v in BD_PART_FLAGS_IDX_FLAG.items()} diff --git a/aif/constants_fallback.py b/aif/constants_fallback.py new file mode 100644 index 0000000..11aaf5e --- /dev/null +++ b/aif/constants_fallback.py @@ -0,0 +1,21 @@ +import parted # https://www.gnu.org/software/parted/api/index.html + + +ARCH_RELENG_KEY = '4AA4767BBC9C4B1D18AE28B77F2D434B9741E8AC' +VERSION = '0.2.0' +EXTERNAL_DEPS = ['blkinfo', + 'gpg', + 'lxml', + 'mdstat', + 'passlib', + 'psutil', + 'pyparted', + 'pyroute2', + 'pytz', + 'requests', + 'validators'] +# PARTED FLAG INDEXING +PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys())) +PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values())) +PARTED_IDX_FLAG = dict(parted.partition.partitionFlag) +PARTED_FLAG_IDX = {v: k for k, v in PARTED_IDX_FLAG.items()} diff --git a/aif/disk/__init__.py b/aif/disk/__init__.py index 7b01166..1f0f126 100644 --- a/aif/disk/__init__.py +++ b/aif/disk/__init__.py @@ -21,3 +21,8 @@ try: from . import mdadm_fallback except ImportError: from . import mdadm_fallback as mdadm + +try: + from . import _common +except ImportError: + pass # GI isn't supported, so we don't even use a fallback. diff --git a/aif/disk/_common.py b/aif/disk/_common.py index 02ef2c7..c193093 100644 --- a/aif/disk/_common.py +++ b/aif/disk/_common.py @@ -2,8 +2,12 @@ import gi gi.require_version('BlockDev', '2.0') from gi.repository import BlockDev, GLib -ps = BlockDev.PluginSpec() -ps.name = BlockDev.Plugin.LVM -ps.so_name = "libbd_lvm.so" +BlockDev.ensure_init([None]) -BlockDev.init([ps]) + +def addBDPlugin(plugin_name): + plugins = BlockDev.get_available_plugin_names() + plugins.append(plugin_name) + plugins = list(set(plugins)) # Deduplicate + spec = BlockDev.plugin_specs_from_names(plugins) + return(BlockDev.ensure_init(spec)) diff --git a/aif/disk/block.py b/aif/disk/block.py index 935f341..fdaa9af 100644 --- a/aif/disk/block.py +++ b/aif/disk/block.py @@ -1,4 +1,133 @@ -from . import _common +import re +## +import parted +import psutil +## +import aif.constants +import aif.disk._common +import aif.utils -_BlockDev = _common.BlockDev +_BlockDev = aif.disk._common.BlockDev + +class Partition(object): + def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype, part_type = None): + # Belive it or not, dear reader, but this *entire method* is just to set attributes. + if tbltype not in ('gpt', 'msdos'): + raise ValueError('{0} must be one of gpt or msdos'.format(tbltype)) + if tbltype == 'msdos' and part_type not in ('primary', 'extended', 'logical'): + raise ValueError(('You must specify if this is a ' + 'primary, extended, or logical partition for msdos partition tables')) + aif.disk._common.addBDPlugin('part') + self.xml = part_xml + self.id = part_xml.attrib['id'] + self.table_type = getattr(_BlockDev.PartTableType, tbltype.upper()) + if tbltype == 'msdos': + # Could technically be _BlockDev.PartTypeReq.NEXT buuuut that doesn't *quite* work + # with our project's structure. + if part_type == 'primary': + self.part_type = _BlockDev.PartTypeReq.NORMAL + elif part_type == 'extended': + self.part_type = _BlockDev.PartTypeReq.EXTENDED + elif part_type == 'logical': + self.part_type = _BlockDev.PartTypeReq.LOGICAL + self.flags = [] + for f in self.xml.findall('partitionFlag'): + # *Technically* we could use e.g. getattr(_BlockDev.PartFlag, f.text.upper()), *but* we lose compat + # with parted's flags if we do that. :| So we do some funky logic both here and in the constants. + if f.text in aif.constants.PARTED_BD_MAP: + flag_id = aif.constants.BD_PART_FLAGS_FLAG_IDX[aif.constants.PARTED_BD_MAP[f.text]] + elif f.text in aif.constants.BD_PART_FLAGS_FRIENDLY: + flag_id = aif.constants.BD_PART_FLAGS_FLAG_IDX[aif.constants.BD_PART_FLAGS_FRIENDLY[f.text]] + else: + continue + self.flags.append(_BlockDev.PartFlag(flag_id)) + self.partnum = partnum + self.fstype = self.xml.attrib['fsType'].lower() + if self.fstype not in aif.constants.PARTED_FSTYPES: # There isn't a way to do this with BlockDev? :| + raise ValueError(('{0} is not a valid partition filesystem type; ' + 'must be one of: {1}').format(self.xml.attrib['fsType'], + ', '.join(sorted(aif.constants.PARTED_FSTYPES)))) + self.disk = diskobj + self.device = self.disk.device # TODO: how to get this in libblockdev? + self.devpath = '{0}{1}'.format(self.device.path, partnum) + self.is_hiformatted = False + sizes = {} + for s in ('start', 'stop'): + x = dict(zip(('from_bgn', 'size', 'type'), + aif.utils.convertSizeUnit(self.xml.attrib[s]))) + sectors = x['size'] + if x['type'] == '%': + sectors = int(self.device.getLength() / x['size']) + else: + sectors = int(aif.utils.size.convertStorage(x['size'], + x['type'], + target = 'B') / self.device.sectorSize) + sizes[s] = (sectors, x['from_bgn']) + if sizes['start'][1] is not None: + if sizes['start'][1]: + self.begin = sizes['start'][0] + 0 + else: + self.begin = self.device.getLength() - sizes['start'][0] # TODO: is there a way to get this in BD? + else: + self.begin = sizes['start'][0] + start_sector + if sizes['stop'][1] is not None: + if sizes['stop'][1]: + self.end = sizes['stop'][0] + 0 + else: + # This *technically* should be - 34, at least for gpt, but the alignment optimizer fixes it for us. + self.end = (self.device.getLength() - 1) - sizes['stop'][0] # TODO: is there a way to get this in BD? + else: + self.end = self.begin + sizes['stop'][0] + # TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit + # in configs. So we manually crunch the numbers and do it all at the end. + # TODO: switch parted objects to BlockDev + # self.geometry = parted.Geometry(device = self.device, + # start = self.begin, + # end = self.end) + # self.filesystem = parted.FileSystem(type = self.fstype, + # geometry = self.geometry) + # self.partition = parted.Partition(disk = diskobj, + # type = self.part_type, + # geometry = self.geometry, + # fs = self.filesystem) + self.part_name = self.xml.attrib.get('name') + + # + # def detect(self): + # pass # TODO; blkinfo? + + +class Disk(object): + def __init__(self, disk_xml): + # TODO: BlockDev.part.set_disk_flag(, + # BlockDev.PartDiskFlag(1), + # True) ?? + # https://lazka.github.io/pgi-docs/BlockDev-2.0/enums.html#BlockDev.PartDiskFlag + # https://unix.stackexchange.com/questions/325886/bios-gpt-do-we-need-a-boot-flag + self.xml = disk_xml + self.devpath = self.xml.attrib['device'] + self.is_lowformatted = None + self.is_hiformatted = None + self.is_partitioned = None + self.partitions = None + self._initDisk() + aif.disk._common.addBDPlugin('part') + + def _initDisk(self): + pass + + def diskFormat(self): + pass + + def getPartitions(self): + pass + + def partFormat(self): + pass + + +class Mount(object): + def __init__(self, mount_xml, partobj): + pass + _common.addBDPlugin('fs') diff --git a/aif/disk/block_fallback.py b/aif/disk/block_fallback.py index f6b40b7..4cb34a7 100644 --- a/aif/disk/block_fallback.py +++ b/aif/disk/block_fallback.py @@ -15,42 +15,12 @@ except ImportError: import parted # https://www.gnu.org/software/parted/api/index.html import psutil ## -from aif.utils import xmlBool, size +import aif.constants +import aif.utils # TODO: https://serverfault.com/questions/356534/ssd-erase-block-size-lvm-pv-on-raw-device-alignment -PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys())) -PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values())) -IDX_FLAG = dict(parted.partition.partitionFlag) -FLAG_IDX = {v: k for k, v in IDX_FLAG.items()} - -# parted lib can do SI or IEC. So can we. -_pos_re = re.compile((r'^(?P-|\+)?\s*' - r'(?P[0-9]+)\s*' - # empty means size in sectors - r'(?P%|{0}|)\s*$'.format('|'.join(size.valid_storage)) - )) - - -def convertSizeUnit(pos): - orig_pos = pos - pos = _pos_re.search(pos) - if pos: - pos_or_neg = (pos.group('pos_or_neg') if pos.group('pos_or_neg') else None) - if pos_or_neg == '+': - from_beginning = True - elif pos_or_neg == '-': - from_beginning = False - else: - from_beginning = pos_or_neg - _size = int(pos.group('size')) - amt_type = pos.group('pct_unit_or_sct').strip() - else: - raise ValueError('Invalid size specified: {0}'.format(orig_pos)) - return((from_beginning, _size, amt_type)) - - class Partition(object): def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype, part_type = None): if tbltype not in ('gpt', 'msdos'): @@ -59,10 +29,10 @@ class Partition(object): raise ValueError(('You must specify if this is a ' 'primary, extended, or logical partition for msdos partition tables')) self.xml = part_xml - self.id = part_xml.attrib['id'] + self.id = self.xml.attrib['id'] self.flags = set() for f in self.xml.findall('partitionFlag'): - if f.text in PARTED_FLAGS: + if f.text in aif.constants.PARTED_FLAGS: self.flags.add(f.text) self.flags = sorted(list(self.flags)) self.partnum = partnum @@ -77,10 +47,10 @@ class Partition(object): else: self.part_type = parted.PARTITION_NORMAL self.fstype = self.xml.attrib['fsType'].lower() - if self.fstype not in PARTED_FSTYPES: + if self.fstype not in aif.constants.PARTED_FSTYPES: raise ValueError(('{0} is not a valid partition filesystem type; ' 'must be one of: {1}').format(self.xml.attrib['fsType'], - ', '.join(sorted(PARTED_FSTYPES)))) + ', '.join(sorted(aif.constants.PARTED_FSTYPES)))) self.disk = diskobj self.device = self.disk.device self.devpath = '{0}{1}'.format(self.device.path, partnum) @@ -88,12 +58,14 @@ class Partition(object): sizes = {} for s in ('start', 'stop'): x = dict(zip(('from_bgn', 'size', 'type'), - convertSizeUnit(self.xml.attrib[s]))) + aif.utils.convertSizeUnit(self.xml.attrib[s]))) sectors = x['size'] if x['type'] == '%': sectors = int(self.device.getLength() / x['size']) else: - sectors = int(size.convertStorage(x['size'], x['type'], target = 'B') / self.device.sectorSize) + sectors = int(aif.utils.size.convertStorage(x['size'], + x['type'], + target = 'B') / self.device.sectorSize) sizes[s] = (sectors, x['from_bgn']) if sizes['start'][1] is not None: if sizes['start'][1]: @@ -122,7 +94,7 @@ class Partition(object): geometry = self.geometry, fs = self.filesystem) for f in self.flags[:]: - flag_id = FLAG_IDX[f] + flag_id = aif.constants.PARTED_FLAG_IDX[f] if self.partition.isFlagAvailable(flag_id): self.partition.setFlag(flag_id) else: @@ -133,16 +105,20 @@ class Partition(object): # https://github.com/dcantrell/pyparted/issues/65 # self.partition.name = self.xml.attrib.get('name') _pedpart = self.partition.getPedPartition() - _pedpart.set_name(self.xml.attrib.get('name')) - - def detect(self): - pass + _pedpart.set_name(self.xml.attrib['name']) + # + # def detect(self): + # pass # TODO; blkinfo? class Disk(object): def __init__(self, disk_xml): self.xml = disk_xml self.devpath = self.xml.attrib['device'] + self.is_lowformatted = None + self.is_hiformatted = None + self.is_partitioned = None + self.partitions = None self._initDisk() def _initDisk(self): @@ -221,6 +197,7 @@ class Disk(object): self.is_partitioned = True return() + class Mount(object): def __init__(self, mount_xml, partobj): self.xml = mount_xml diff --git a/aif/utils.py b/aif/utils.py index 799f7ad..b862f9c 100644 --- a/aif/utils.py +++ b/aif/utils.py @@ -1,9 +1,13 @@ import os +import re + def hasBin(binary_name): paths = [] for p in os.environ.get('PATH', '/usr/bin:/bin').split(':'): - os.listdir(os.path.realpath(p)) + if binary_name in os.listdir(os.path.realpath(p)): + return(os.path.join(p, binary_name)) + return(False) def xmlBool(xmlobj): @@ -181,3 +185,30 @@ class _Sizer(object): size = _Sizer() + + +# We do this as base level so they aren't compiled on every invocation/instantiation. +# parted lib can do SI or IEC. So can we. +_pos_re = re.compile((r'^(?P-|\+)?\s*' + r'(?P[0-9]+)\s*' + # empty means size in sectors + r'(?P%|{0}|)\s*$'.format('|'.join(size.valid_storage)) + )) + + +def convertSizeUnit(pos): + orig_pos = pos + pos = _pos_re.search(pos) + if pos: + pos_or_neg = (pos.group('pos_or_neg') if pos.group('pos_or_neg') else None) + if pos_or_neg == '+': + from_beginning = True + elif pos_or_neg == '-': + from_beginning = False + else: + from_beginning = pos_or_neg + _size = int(pos.group('size')) + amt_type = pos.group('pct_unit_or_sct').strip() + else: + raise ValueError('Invalid size specified: {0}'.format(orig_pos)) + return((from_beginning, _size, amt_type))