From d4de31dd6717cccd703b2a8a8cf95295bbc20b3c Mon Sep 17 00:00:00 2001 From: brent s Date: Wed, 6 Nov 2019 02:21:04 -0500 Subject: [PATCH] filesystem and mounting done, other minor tweaks. need to do lvm, mdadm, luks --- aif.xsd | 3 + aif/constants_fallback.py | 39 ++++++ aif/disk/block.py | 220 ++++++++++++++++++++++---------- aif/disk/block_fallback.py | 53 ++++---- aif/disk/filesystem.py | 113 +++++++++++++++- aif/disk/filesystem_fallback.py | 118 ++++++++++------- aif/utils.py | 86 ++++++++++--- examples/aif.xml | 31 +++-- 8 files changed, 492 insertions(+), 171 deletions(-) diff --git a/aif.xsd b/aif.xsd index 5e07c97..d949106 100644 --- a/aif.xsd +++ b/aif.xsd @@ -262,6 +262,7 @@ + @@ -384,6 +385,7 @@ + @@ -406,6 +408,7 @@ + diff --git a/aif/constants_fallback.py b/aif/constants_fallback.py index fc290b8..ab41d27 100644 --- a/aif/constants_fallback.py +++ b/aif/constants_fallback.py @@ -19,6 +19,44 @@ EXTERNAL_DEPS = ['blkinfo', 'validators'] # PARTED FLAG INDEXING PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys())) +PARTED_FSTYPES_GUIDS = {'affs0': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'affs1': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'affs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'affs3': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'affs4': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'affs5': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'affs6': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'affs7': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'amufs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'amufs0': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'amufs1': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'amufs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'amufs3': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'amufs4': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'amufs5': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'apfs1': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'apfs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'asfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'btrfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'ext2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'ext3': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'ext4': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'fat16': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'), + 'fat32': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'), + 'hfs': uuid.UUID(hex = '48465300-0000-11AA-AA11-00306543ECAC'), + 'hfs+': uuid.UUID(hex = '48465300-0000-11AA-AA11-00306543ECAC'), + 'hfsx': uuid.UUID(hex = '48465300-0000-11AA-AA11-00306543ECAC'), + 'hp-ufs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'jfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'linux-swap(v0)': uuid.UUID(hex = '0657FD6D-A4AB-43C4-84E5-0933C84B4F4F'), + 'linux-swap(v1)': uuid.UUID(hex = '0657FD6D-A4AB-43C4-84E5-0933C84B4F4F'), + 'nilfs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'ntfs': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'), + 'reiserfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'sun-ufs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'swsusp': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'), + 'udf': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'), + 'xfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4')} PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values())) PARTED_IDX_FLAG = dict(parted.partition.partitionFlag) PARTED_FLAG_IDX = {v: k for k, v in PARTED_IDX_FLAG.items()} @@ -125,6 +163,7 @@ GPT_FSTYPE_GUIDS = ((1, 'EFI System', uuid.UUID(hex = 'C12A7328-F81F-11D2-BA4B-0 (87, 'Plan 9 partition', uuid.UUID(hex = 'C91818F9-8025-47AF-89D2-F030D7000C2C')), (88, 'HiFive Unleashed FSBL', uuid.UUID(hex = '5B193300-FC78-40CD-8002-E86C45580B47')), (89, 'HiFive Unleashed BBL', uuid.UUID(hex = '2E54B353-1271-4842-806F-E436D6AF6985'))) +GPT_GUID_IDX = {k[2]: k[0] for k in GPT_FSTYPE_GUIDS} # MSDOS FSTYPES IDENTIFIERS # Second verse, same as the first - kind of. The msdos type identifers just use a byte identifier rather than UUID. # https://git.kernel.org/pub/scm/utils/util-linux/util-linux.git/plain/include/pt-mbr-partnames.h diff --git a/aif/disk/block.py b/aif/disk/block.py index 74335be..3d45ab4 100644 --- a/aif/disk/block.py +++ b/aif/disk/block.py @@ -1,6 +1,7 @@ -import re +import os +import uuid ## -import parted +import blkinfo import psutil # Do I need this if I can have libblockdev's mounts API? Is there a way to get current mounts? ## import aif.constants @@ -10,7 +11,7 @@ from . import _common _BlockDev = _common.BlockDev - +# TODO: LOGGING! class Partition(object): def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype, part_type = None): # Belive it or not, dear reader, but this *entire method* is just to set attributes. @@ -32,7 +33,49 @@ class Partition(object): self.part_type = _BlockDev.PartTypeReq.EXTENDED elif part_type == 'logical': self.part_type = _BlockDev.PartTypeReq.LOGICAL + elif tbltype == 'gpt': + self.part_type = _BlockDev.PartTypeReq.NORMAL self.flags = [] + self.partnum = partnum + self.fs_type = self.xml.attrib['fsType'] + self.disk = diskobj + self.device = self.disk.path + self.devpath = '{0}{1}'.format(self.device, self.partnum) + self.is_hiformatted = False + sizes = {} + for s in ('start', 'stop'): + x = dict(zip(('from_bgn', 'size', 'type'), + aif.utils.convertSizeUnit(self.xml.attrib[s]))) + sectors = x['size'] + if x['type'] == '%': + sectors = int(int(self.disk.size / self.disk.sector_size) * (0.01 * x['size'])) + else: + sectors = int(aif.utils.size.convertStorage(x['size'], + x['type'], + target = 'B') / self.disk.sector_size) + sizes[s] = (sectors, x['from_bgn']) + if sizes['start'][1] is not None: + if sizes['start'][1]: + self.begin = sizes['start'][0] + 0 + else: + self.begin = int(self.disk.size / self.disk.sector_size) - sizes['start'][0] + else: + self.begin = sizes['start'][0] + start_sector + if sizes['stop'][1] is not None: + if sizes['stop'][1]: + self.end = sizes['stop'][0] + 0 + else: + # This *technically* should be - 34, at least for gpt, but the alignment optimizer fixes it for us. + self.end = (int(self.disk.size / self.disk.sector_size) - 1) - sizes['stop'][0] + else: + self.end = self.begin + sizes['stop'][0] + self.size = (self.end - self.begin) + self.part_name = self.xml.attrib.get('name') + self.partition = None + self._initFlags() + self._initFstype() + + def _initFlags(self): for f in self.xml.findall('partitionFlag'): # *Technically* we could use e.g. getattr(_BlockDev.PartFlag, f.text.upper()), *but* we lose compat # with parted's flags if we do that. :| So we do some funky logic both here and in the constants. @@ -43,56 +86,40 @@ class Partition(object): else: continue self.flags.append(_BlockDev.PartFlag(flag_id)) - self.partnum = partnum - self.fstype = self.xml.attrib['fsType'].lower() - if self.fstype not in aif.constants.PARTED_FSTYPES: # There isn't a way to do this with BlockDev? :| - raise ValueError(('{0} is not a valid partition filesystem type; ' - 'must be one of: {1}').format(self.xml.attrib['fsType'], - ', '.join(sorted(aif.constants.PARTED_FSTYPES)))) - self.disk = diskobj - self.device = self.disk.path - self.devpath = '{0}{1}'.format(self.device.path, partnum) - self.is_hiformatted = False - sizes = {} - for s in ('start', 'stop'): - x = dict(zip(('from_bgn', 'size', 'type'), - aif.utils.convertSizeUnit(self.xml.attrib[s]))) - sectors = x['size'] - if x['type'] == '%': - sectors = int(self.device.getLength() / x['size']) - else: - sectors = int(aif.utils.size.convertStorage(x['size'], - x['type'], - target = 'B') / self.device.sectorSize) - sizes[s] = (sectors, x['from_bgn']) - if sizes['start'][1] is not None: - if sizes['start'][1]: - self.begin = sizes['start'][0] + 0 - else: - self.begin = self.device.getLength() - sizes['start'][0] # TODO: is there a way to get this in BD? + return() + + def _initFstype(self): + _err = ('{0} is not a valid partition filesystem type; ' + 'must be one of {1} or an fdisk-compatible GPT GUID').format( + self.xml.attrib['fsType'], + ', '.join(sorted(aif.constants.PARTED_FSTYPES))) + if self.fs_type in aif.constants.PARTED_FSTYPES_GUIDS.keys(): + self.fs_type = aif.constants.PARTED_FSTYPES_GUIDS[self.fs_type] else: - self.begin = sizes['start'][0] + start_sector - if sizes['stop'][1] is not None: - if sizes['stop'][1]: - self.end = sizes['stop'][0] + 0 - else: - # This *technically* should be - 34, at least for gpt, but the alignment optimizer fixes it for us. - self.end = (self.device.getLength() - 1) - sizes['stop'][0] # TODO: is there a way to get this in BD? - else: - self.end = self.begin + sizes['stop'][0] - # TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit - # in configs. So we manually crunch the numbers and do it all at the end. - # TODO: switch parted objects to BlockDev - # self.geometry = parted.Geometry(device = self.device, - # start = self.begin, - # end = self.end) - # self.filesystem = parted.FileSystem(type = self.fstype, - # geometry = self.geometry) - # self.partition = parted.Partition(disk = diskobj, - # type = self.part_type, - # geometry = self.geometry, - # fs = self.filesystem) - self.part_name = self.xml.attrib.get('name') + try: + self.fs_type = uuid.UUID(hex = self.fs_type) + except ValueError: + raise ValueError(_err) + if self.fs_type not in aif.constants.GPT_GUID_IDX.keys(): + raise ValueError(_err) + return() + + def format(self): + # This is a safeguard. We do *not* want to partition a disk that is mounted. + aif.utils.checkMounted(self.devpath) + self.partition = _BlockDev.part.create_part(self.device, + self.part_type, + self.begin, + self.size, + _BlockDev.PartAlign.OPTIMAL) + self.devpath = self.partition.path + _BlockDev.part.set_part_type(self.device, self.devpath, str(self.fs_type).upper()) + if self.part_name: + _BlockDev.part.set_part_name(self.device, self.devpath, self.part_name) + if self.flags: + for f in self.flags: + _BlockDev.part.set_part_flag(self.device, self.devpath, f, True) + return() # # def detect(self): @@ -101,34 +128,89 @@ class Partition(object): class Disk(object): def __init__(self, disk_xml): - # TODO: BlockDev.part.set_disk_flag(, - # BlockDev.PartDiskFlag(1), - # True) ?? - # https://lazka.github.io/pgi-docs/BlockDev-2.0/enums.html#BlockDev.PartDiskFlag - # https://unix.stackexchange.com/questions/325886/bios-gpt-do-we-need-a-boot-flag self.xml = disk_xml - self.devpath = self.xml.attrib['device'] + self.devpath = os.path.realpath(self.xml.attrib['device']) + aif.disk._common.addBDPlugin('part') self.is_lowformatted = None self.is_hiformatted = None self.is_partitioned = None self.partitions = None self._initDisk() - aif.disk._common.addBDPlugin('part') def _initDisk(self): - pass + if self.devpath == 'auto': + self.devpath = '/dev/{0}'.format(blkinfo.BlkDiskInfo().get_disks()[0]['kname']) + if not os.path.isfile(self.devpath): + raise ValueError('{0} does not exist; please specify an explicit device path'.format(self.devpath)) + self.table_type = self.xml.attrib.get('diskFormat', 'gpt').lower() + if self.table_type in ('bios', 'mbr', 'dos', 'msdos'): + self.table_type = _BlockDev.PartTableType.MSDOS + elif self.table_type == 'gpt': + self.table_type = _BlockDev.PartTableType.GPT + else: + raise ValueError(('Disk format {0} is not valid for this architecture;' + 'must be one of: gpt or msdos'.format(self.table_type))) + self.device = self.disk = _BlockDev.part.get_disk_spec(self.devpath) + self.is_lowformatted = False + self.is_hiformatted = False + self.is_partitioned = False + self.partitions = [] + return() def diskFormat(self): - pass + if self.is_lowformatted: + return () + # This is a safeguard. We do *not* want to low-format a disk that is mounted. + aif.utils.checkMounted(self.devpath) + # TODO: BlockDev.part.set_disk_flag(, + # BlockDev.PartDiskFlag(1), + # True) ?? + # https://lazka.github.io/pgi-docs/BlockDev-2.0/enums.html#BlockDev.PartDiskFlag + # https://unix.stackexchange.com/questions/325886/bios-gpt-do-we-need-a-boot-flag + _BlockDev.part.create_table(self.devpath, self.table_type, True) + self.is_lowformatted = True + self.is_partitioned = False + return() def getPartitions(self): - pass + # For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment + # optimizer fixes it for us automatically. + # But for DOS tables, it's required. + if self.table_type == 'msdos': + start_sector = 2048 + else: + start_sector = 0 + self.partitions = [] + xml_partitions = self.xml.findall('part') + for idx, part in enumerate(xml_partitions): + partnum = idx + 1 + if self.table_type == 'gpt': + p = Partition(part, self.disk, start_sector, partnum, self.table_type) + else: + parttype = 'primary' + if len(xml_partitions) > 4: + if partnum == 4: + parttype = 'extended' + elif partnum > 4: + parttype = 'logical' + p = Partition(part, self.disk, start_sector, partnum, self.table_type, part_type = parttype) + start_sector = p.end + 1 + self.partitions.append(p) + return() def partFormat(self): - pass - - -class Mount(object): - def __init__(self, mount_xml, partobj): - pass - _common.addBDPlugin('fs') + if self.is_partitioned: + return() + if not self.is_lowformatted: + self.diskFormat() + # This is a safeguard. We do *not* want to partition a disk that is mounted. + aif.utils.checkMounted(self.devpath) + if not self.partitions: + self.getPartitions() + if not self.partitions: + return() + for p in self.partitions: + p.format() + p.is_hiformatted = True + self.is_partitioned = True + return () diff --git a/aif/disk/block_fallback.py b/aif/disk/block_fallback.py index 9b1331c..d85e408 100644 --- a/aif/disk/block_fallback.py +++ b/aif/disk/block_fallback.py @@ -3,6 +3,7 @@ # https://github.com/dcantrell/pyparted/blob/master/examples/query_device_capacity.py # TODO: Remember to replicate genfstab behaviour. +import os import re try: # https://stackoverflow.com/a/34812552/733214 @@ -12,6 +13,7 @@ except ImportError: # We should never get here. util-linux is part of core (base) in Arch and uses "libmount". import pylibmount as mount ## +import blkinfo import parted # https://www.gnu.org/software/parted/api/index.html import psutil ## @@ -48,14 +50,14 @@ class Partition(object): self.part_type = parted.PARTITION_LOGICAL else: self.part_type = parted.PARTITION_NORMAL - self.fstype = self.xml.attrib['fsType'].lower() - if self.fstype not in aif.constants.PARTED_FSTYPES: + self.fs_type = self.xml.attrib['fsType'].lower() + if self.fs_type not in aif.constants.PARTED_FSTYPES: raise ValueError(('{0} is not a valid partition filesystem type; ' 'must be one of: {1}').format(self.xml.attrib['fsType'], ', '.join(sorted(aif.constants.PARTED_FSTYPES)))) self.disk = diskobj self.device = self.disk.device - self.devpath = '{0}{1}'.format(self.device.path, partnum) + self.devpath = '{0}{1}'.format(self.device.path, self.partnum) self.is_hiformatted = False sizes = {} for s in ('start', 'stop'): @@ -63,7 +65,7 @@ class Partition(object): aif.utils.convertSizeUnit(self.xml.attrib[s]))) sectors = x['size'] if x['type'] == '%': - sectors = int(self.device.getLength() / x['size']) + sectors = int(self.device.getLength() * (0.01 * x['size'])) else: sectors = int(aif.utils.size.convertStorage(x['size'], x['type'], @@ -89,7 +91,7 @@ class Partition(object): self.geometry = parted.Geometry(device = self.device, start = self.begin, end = self.end) - self.filesystem = parted.FileSystem(type = self.fstype, + self.filesystem = parted.FileSystem(type = self.fs_type, geometry = self.geometry) self.partition = parted.Partition(disk = diskobj, type = self.part_type, @@ -116,7 +118,8 @@ class Partition(object): class Disk(object): def __init__(self, disk_xml): self.xml = disk_xml - self.devpath = self.xml.attrib['device'] + self.id = self.xml.attrib['id'] + self.devpath = os.path.realpath(self.xml.attrib['device']) self.is_lowformatted = None self.is_hiformatted = None self.is_partitioned = None @@ -124,15 +127,19 @@ class Disk(object): self._initDisk() def _initDisk(self): - self.tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower() - if self.tabletype in ('bios', 'mbr', 'dos'): - self.tabletype = 'msdos' + if self.devpath == 'auto': + self.devpath = '/dev/{0}'.format(blkinfo.BlkDiskInfo().get_disks()[0]['kname']) + if not os.path.isfile(self.devpath): + raise ValueError('{0} does not exist; please specify an explicit device path'.format(self.devpath)) + self.table_type = self.xml.attrib.get('diskFormat', 'gpt').lower() + if self.table_type in ('bios', 'mbr', 'dos'): + self.table_type = 'msdos' validlabels = parted.getLabels() - if self.tabletype not in validlabels: + if self.table_type not in validlabels: raise ValueError(('Disk format {0} is not valid for this architecture;' - 'must be one of: {1}'.format(self.tabletype, ', '.join(list(validlabels))))) + 'must be one of: {1}'.format(self.table_type, ', '.join(list(validlabels))))) self.device = parted.getDevice(self.devpath) - self.disk = parted.freshDisk(self.device, self.tabletype) + self.disk = parted.freshDisk(self.device, self.table_type) self.is_lowformatted = False self.is_hiformatted = False self.is_partitioned = False @@ -143,9 +150,7 @@ class Disk(object): if self.is_lowformatted: return() # This is a safeguard. We do *not* want to low-format a disk that is mounted. - for p in psutil.disk_partitions(all = True): - if self.devpath in p: - raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) + aif.utils.checkMounted(self.devpath) self.disk.deleteAllPartitions() self.disk.commit() self.is_lowformatted = True @@ -156,7 +161,7 @@ class Disk(object): # For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment # optimizer fixes it for us automatically. # But for DOS tables, it's required. - if self.tabletype == 'msdos': + if self.table_type == 'msdos': start_sector = 2048 else: start_sector = 0 @@ -164,8 +169,8 @@ class Disk(object): xml_partitions = self.xml.findall('part') for idx, part in enumerate(xml_partitions): partnum = idx + 1 - if self.tabletype == 'gpt': - p = Partition(part, self.disk, start_sector, partnum, self.tabletype) + if self.table_type == 'gpt': + p = Partition(part, self.disk, start_sector, partnum, self.table_type) else: parttype = 'primary' if len(xml_partitions) > 4: @@ -173,7 +178,7 @@ class Disk(object): parttype = 'extended' elif partnum > 4: parttype = 'logical' - p = Partition(part, self.disk, start_sector, partnum, self.tabletype, part_type = parttype) + p = Partition(part, self.disk, start_sector, partnum, self.table_type, part_type = parttype) start_sector = p.end + 1 self.partitions.append(p) return() @@ -184,9 +189,7 @@ class Disk(object): if not self.is_lowformatted: self.diskFormat() # This is a safeguard. We do *not* want to partition a disk that is mounted. - for p in psutil.disk_partitions(all = True): - if self.devpath in p: - raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) + aif.utils.checkMounted(self.devpath) if not self.partitions: self.getPartitions() if not self.partitions: @@ -198,9 +201,3 @@ class Disk(object): p.is_hiformatted = True self.is_partitioned = True return() - - -class Mount(object): - def __init__(self, mount_xml, partobj): - self.xml = mount_xml - pass diff --git a/aif/disk/filesystem.py b/aif/disk/filesystem.py index 4736de8..d49739c 100644 --- a/aif/disk/filesystem.py +++ b/aif/disk/filesystem.py @@ -1,3 +1,114 @@ +import os +import subprocess +## +import psutil +## +import aif.disk.block as block +import aif.disk.luks as luks +import aif.disk.lvm as lvm +import aif.disk.mdadm as mdadm +import aif.utils from . import _common -BlockDev = _common.BlockDev +_BlockDev = _common.BlockDev + + +FS_FSTYPES = aif.utils.kernelFilesystems() + + +class FS(object): + def __init__(self, fs_xml, sourceobj): + # http://storaged.org/doc/udisks2-api/latest/gdbus-org.freedesktop.UDisks2.Filesystem.html#gdbus-interface-org-freedesktop-UDisks2-Filesystem.top_of_page + # http://storaged.org/doc/udisks2-api/latest/ ? + self.xml = fs_xml + self.id = self.xml.attrib['id'] + if not isinstance(sourceobj, (block.Disk, + block.Partition, + luks.LUKS, + lvm.LV, + mdadm.Array)): + raise ValueError(('sourceobj must be of type ' + 'aif.disk.block.Partition, ' + 'aif.disk.luks.LUKS, ' + 'aif.disk.lvm.LV, or' + 'aif.disk.mdadm.Array')) + self.source = sourceobj + self.devpath = sourceobj.devpath + self.formatted = False + self.fstype = self.xml.attrib.get('type') + if self.fstype not in FS_FSTYPES: + raise ValueError('{0} is not a supported filesystem type on this system'.format(self.fstype)) + + def format(self): + if self.formatted: + return () + # This is a safeguard. We do *not* want to high-format a disk that is mounted. + aif.utils.checkMounted(self.devpath) + # TODO: Can I format with DBus/gobject-introspection? I feel like I *should* be able to, but BlockDev's fs + # plugin is *way* too limited in terms of filesystems and UDisks doesn't let you format that high-level. + # TODO! Logging + cmd = ['mkfs', + '-t', self.fstype] + for o in self.xml.findall('opt'): + cmd.append(o.attrib['name']) + if o.text: + cmd.append(o.text) + cmd.append(self.devpath) + subprocess.run(cmd) + self.formatted = True + return() + + +class Mount(object): + # http://storaged.org/doc/udisks2-api/latest/gdbus-org.freedesktop.UDisks2.Filesystem.html#gdbus-method-org-freedesktop-UDisks2-Filesystem.Mount + def __init__(self, mount_xml, fsobj): + self.xml = mount_xml + if not isinstance(fsobj, FS): + raise ValueError('partobj must be of type aif.disk.filesystem.FS') + _common.addBDPlugin('fs') # We *could* use the UDisks dbus to mount too, but best to stay within libblockdev. + self.id = self.xml.attrib['id'] + self.fs = fsobj + self.source = self.fs.devpath + self.target = os.path.realpath(self.xml.attrib['target']) + self.opts = {} + for o in self.xml.findall('opt'): + self.opts[o.attrib['name']] = o.text + self.mounted = False + + def _parseOpts(self): + opts = [] + for k, v in self.opts.items(): + if v and v is not True: # Python's boolean determination is weird sometimes. + opts.append('{0}={1}'.format(k, v)) + else: + opts.append(k) + return(opts) + + def mount(self): + if self.mounted: + return() + os.makedirs(self.target, exist_ok = True) + opts = self._parseOpts() + _BlockDev.fs.mount(self.source, + self.target, + self.fs.fstype, + (','.join(opts) if opts else None)) + self.mounted = True + return() + + def unmount(self, lazy = False, force = False): + self.updateMount() + if not self.mounted and not force: + return() + _BlockDev.fs.unmount(self.target, + lazy, + force) + self.mounted = False + return() + + def updateMount(self): + if self.source in [p.device for p in psutil.disk_partitions(all = True)]: + self.mounted = True + else: + self.mounted = False + return() diff --git a/aif/disk/filesystem_fallback.py b/aif/disk/filesystem_fallback.py index 06127c9..ee1bfa3 100644 --- a/aif/disk/filesystem_fallback.py +++ b/aif/disk/filesystem_fallback.py @@ -1,5 +1,4 @@ import os -import re import subprocess ## import psutil @@ -8,56 +7,26 @@ import aif.disk.block_fallback as block import aif.disk.luks_fallback as luks import aif.disk.lvm_fallback as lvm import aif.disk.mdadm_fallback as mdadm +import aif.utils -# I wish there was a better way of doing this. -# https://unix.stackexchange.com/a/98680 -FS_FSTYPES = [] -with open('/proc/filesystems', 'r') as fh: - for line in fh.readlines(): - l = [i.strip() for i in line.split()] - if not l: - continue - if len(l) == 1: - FS_FSTYPES.append(l[0]) - else: - FS_FSTYPES.append(l[1]) -_mod_dir = os.path.join('/lib/modules', - os.uname().release, - 'kernel/fs') -_strip_mod_suffix = re.compile(r'(?P)\.ko(\.(x|g)?z)?$', re.IGNORECASE) -try: - for i in os.listdir(_mod_dir): - path = os.path.join(_mod_dir, i) - fs_name = None - if os.path.isdir(path): - fs_name = i - elif os.path.isfile(path): - mod_name = _strip_mod_suffix.search(i) - fs_name = mod_name.group('fsname') - if fs_name: - # The kernel *probably* has autoloading enabled, but in case it doesn't... - # TODO: logging! - if os.getuid() == 0: - subprocess.run(['modprobe', fs_name]) - FS_FSTYPES.append(fs_name) -except FileNotFoundError: - # We're running on a kernel that doesn't have modules - pass + +FS_FSTYPES = aif.utils.kernelFilesystems() class FS(object): def __init__(self, fs_xml, sourceobj): self.xml = fs_xml - if not isinstance(sourceobj, (aif.disk.block_fallback.Disk, - aif.disk.block_fallback.Partition, - aif.disk.luks_fallback.LUKS, - aif.disk.lvm_fallback.LV, - aif.disk.mdadm_fallback.Array)): + if not isinstance(sourceobj, (block.Disk, + block.Partition, + luks.LUKS, + lvm.LV, + mdadm.Array)): raise ValueError(('sourceobj must be of type ' 'aif.disk.block.Partition, ' 'aif.disk.luks.LUKS, ' 'aif.disk.lvm.LV, or' 'aif.disk.mdadm.Array')) + self.id = self.xml.attrib['id'] self.source = sourceobj self.devpath = sourceobj.devpath self.formatted = False @@ -67,10 +36,7 @@ class FS(object): if self.formatted: return () # This is a safeguard. We do *not* want to high-format a disk that is mounted. - for p in psutil.disk_partitions(all = True): - if self.devpath in p: - raise RuntimeError(('{0} is mounted;' - 'we are cowardly refusing to apply a filesystem to it').format(self.devpath)) + aif.utils.checkMounted(self.devpath) # TODO! Logging cmd = ['mkfs', '-t', self.fstype] @@ -80,5 +46,67 @@ class FS(object): cmd.append(o.text) cmd.append(self.devpath) subprocess.run(cmd) - self.is_hiformatted = True + self.formatted = True + return() + + +class Mount(object): + def __init__(self, mount_xml, fsobj): + self.xml = mount_xml + self.id = self.xml.attrib['id'] + if not isinstance(fsobj, FS): + raise ValueError('partobj must be of type aif.disk.filesystem.FS') + self.id = self.xml.attrib['id'] + self.fs = fsobj + self.source = self.fs.devpath + self.target = os.path.realpath(self.xml.attrib['target']) + self.opts = {} + for o in self.xml.findall('opt'): + self.opts[o.attrib['name']] = o.text + self.mounted = False + + def _parseOpts(self): + opts = [] + for k, v in self.opts.items(): + if v and v is not True: # Python's boolean determination is weird sometimes. + opts.append('{0}={1}'.format(k, v)) + else: + opts.append(k) + return(opts) + + def mount(self): + if self.mounted: + return() + os.makedirs(self.target, exist_ok = True) + opts = self._parseOpts() + # TODO: logging + cmd = ['/usr/bin/mount', + '--types', self.fs.fstype] + if opts: + cmd.extend(['--options', ','.join(opts)]) + cmd.extend([self.source, self.target]) + subprocess.run(cmd) + self.mounted = True + return() + + def unmount(self, lazy = False, force = False): + self.updateMount() + if not self.mounted and not force: + return() + # TODO: logging + cmd = ['/usr/bin/umount'] + if lazy: + cmd.append('--lazy') + if force: + cmd.append('--force') + cmd.append(self.target) + subprocess.run(cmd) + self.mounted = False + return() + + def updateMount(self): + if self.source in [p.device for p in psutil.disk_partitions(all = True)]: + self.mounted = True + else: + self.mounted = False return() diff --git a/aif/utils.py b/aif/utils.py index b862f9c..7ffadbe 100644 --- a/aif/utils.py +++ b/aif/utils.py @@ -1,25 +1,14 @@ import os import re +import subprocess +## +import psutil -def hasBin(binary_name): - paths = [] - for p in os.environ.get('PATH', '/usr/bin:/bin').split(':'): - if binary_name in os.listdir(os.path.realpath(p)): - return(os.path.join(p, binary_name)) - return(False) - - -def xmlBool(xmlobj): - # https://bugs.launchpad.net/lxml/+bug/1850221 - if isinstance(xmlobj, bool): - return (xmlobj) - if xmlobj.lower() in ('1', 'true'): - return(True) - elif xmlobj.lower() in ('0', 'false'): - return(False) - else: - return(None) +def checkMounted(devpath): + if devpath in [p.device for p in psutil.disk_partitions(all = True)]: + raise RuntimeError('{0} is mounted; we are cowardly refusing to destructive operations on it'.format(devpath)) + return() def collapseKeys(d, keylist = None): @@ -45,6 +34,65 @@ def collapseValues(d, vallist = None): return(vallist) +def hasBin(binary_name): + paths = [] + for p in os.environ.get('PATH', '/usr/bin:/bin').split(':'): + if binary_name in os.listdir(os.path.realpath(p)): + return(os.path.join(p, binary_name)) + return(False) + + +def kernelFilesystems(): + # I wish there was a better way of doing this. + # https://unix.stackexchange.com/a/98680 + FS_FSTYPES = ['swap'] + with open('/proc/filesystems', 'r') as fh: + for line in fh.readlines(): + l = [i.strip() for i in line.split()] + if not l: + continue + if len(l) == 1: + FS_FSTYPES.append(l[0]) + else: + FS_FSTYPES.append(l[1]) + _mod_dir = os.path.join('/lib/modules', + os.uname().release, + 'kernel/fs') + _strip_mod_suffix = re.compile(r'(?P)\.ko(\.(x|g)?z)?$', re.IGNORECASE) + try: + for i in os.listdir(_mod_dir): + path = os.path.join(_mod_dir, i) + fs_name = None + if os.path.isdir(path): + fs_name = i + elif os.path.isfile(path): + mod_name = _strip_mod_suffix.search(i) + fs_name = mod_name.group('fsname') + if fs_name: + # The kernel *probably* has autoloading enabled, but in case it doesn't... + # TODO: logging! + if os.getuid() == 0: + subprocess.run(['modprobe', fs_name]) + FS_FSTYPES.append(fs_name) + except FileNotFoundError: + # We're running on a kernel that doesn't have modules + pass + FS_FSTYPES = sorted(list(set(FS_FSTYPES))) + return(FS_FSTYPES) + + +def xmlBool(xmlobj): + # https://bugs.launchpad.net/lxml/+bug/1850221 + if isinstance(xmlobj, bool): + return (xmlobj) + if xmlobj.lower() in ('1', 'true'): + return(True) + elif xmlobj.lower() in ('0', 'false'): + return(False) + else: + return(None) + + class _Sizer(object): def __init__(self): # We use different methods for converting between storage and BW, and different multipliers for each subtype. @@ -188,6 +236,7 @@ size = _Sizer() # We do this as base level so they aren't compiled on every invocation/instantiation. +# Unfortunately it has to be at the bottom so we can call the instantiated _Sizer() class. # parted lib can do SI or IEC. So can we. _pos_re = re.compile((r'^(?P-|\+)?\s*' r'(?P[0-9]+)\s*' @@ -212,3 +261,4 @@ def convertSizeUnit(pos): else: raise ValueError('Invalid size specified: {0}'.format(orig_pos)) return((from_beginning, _size, amt_type)) + diff --git a/examples/aif.xml b/examples/aif.xml index 274b8bc..d15741b 100644 --- a/examples/aif.xml +++ b/examples/aif.xml @@ -5,7 +5,7 @@ version="v2_rewrite"> - + @@ -23,7 +23,7 @@ raid - + swap @@ -38,7 +38,8 @@ - + secrets1 @@ -77,19 +78,29 @@ - + 32 ESP - + seekrit + + + - - + + + lzo @@ -98,10 +109,10 @@ 5 / - + - - + +