From 108588827af6f71796609e2d77aa3f11903cbed4 Mon Sep 17 00:00:00 2001 From: brent s Date: Wed, 9 Oct 2019 07:18:10 -0400 Subject: [PATCH] still doing some work but checking in what i have so far --- aif.xsd | 72 ++++++++++---------- aif/aif_util.py | 9 +++ aif/disk.py | 150 +++++++++++++++++++++++++++++++++++++++++- aif/envsetup.py | 5 +- docs/README.adoc | 11 ++-- docs/TODO | 2 + docs/examples/aif.xml | 77 ++++++++++++++-------- setup.py | 3 +- 8 files changed, 259 insertions(+), 70 deletions(-) create mode 100644 aif/aif_util.py diff --git a/aif.xsd b/aif.xsd index ceb5b8e..5f007e3 100644 --- a/aif.xsd +++ b/aif.xsd @@ -1,8 +1,10 @@ + targetNamespace="http://aif-ng.io/" + xmlns="http://aif-ng.io/" + xmlns:aif="http://aif-ng.io/" + elementFormDefault="qualified" + attributeFormDefault="unqualified"> See https://aif.square-r00t.net/ for more information about this project. @@ -148,22 +150,24 @@ + - - - + + + - + - - + + - + @@ -171,13 +175,13 @@ - + - - + + - + @@ -194,10 +198,10 @@ - - - - + + + + @@ -205,7 +209,7 @@ - + @@ -229,30 +233,30 @@ - + - + - + - + - + - + @@ -262,7 +266,7 @@ - + @@ -288,7 +292,7 @@ - + @@ -297,11 +301,11 @@ - + - + @@ -325,7 +329,7 @@ - + @@ -337,19 +341,19 @@ - + - + - + - + diff --git a/aif/aif_util.py b/aif/aif_util.py new file mode 100644 index 0000000..7e9d648 --- /dev/null +++ b/aif/aif_util.py @@ -0,0 +1,9 @@ +def xmlBool(xmlobj): + if isinstance(xmlobj, bool): + return (xmlobj) + if xmlobj.lower() in ('1', 'true'): + return(True) + elif xmlobj.lower() in ('0', 'false'): + return(False) + else: + return(None) \ No newline at end of file diff --git a/aif/disk.py b/aif/disk.py index 905bc15..ec47a6c 100644 --- a/aif/disk.py +++ b/aif/disk.py @@ -3,6 +3,9 @@ # https://github.com/dcantrell/pyparted/blob/master/examples/query_device_capacity.py # TODO: Remember to replicate genfstab behaviour. +import os +import re +import subprocess try: # https://stackoverflow.com/a/34812552/733214 # https://github.com/karelzak/util-linux/blob/master/libmount/python/test_mount_context.py#L6 @@ -11,5 +14,150 @@ except ImportError: # We should never get here. util-linux is part of core (base) in Arch and uses "libmount". import pylibmount as mount ## -import parted +import blkinfo +import parted # https://www.gnu.org/software/parted/api/index.html import psutil +## +from .aif_util import xmlBool + + +# parted lib can do SI or IEC (see table to right at https://en.wikipedia.org/wiki/Binary_prefix) +# We bit-shift to do conversions: +# https://stackoverflow.com/a/12912296/733214 +# https://stackoverflow.com/a/52684562/733214 +_units = {'B': 0, + 'kB': 7, + 'MB': 17, + 'GB': 27, + 'TB': 37, + 'KiB': 10, + 'MiB': 20, + 'GiB': 30, + 'TiB': 40} +_pos_re = re.compile((r'^(?P-|\+)?\s*' + r'(?P[0-9]+)\s*' + # empty means size in sectors + r'(?P%|[{0}]|)\s*$'.format(''.join(list(_units.keys())))), + re.IGNORECASE) + + +def convertSizeUnit(pos): + orig_pos = pos + pos = _pos_re.search(pos) + if pos: + pos_or_neg = (pos.group('pos_or_neg') if pos.group('pos_or_neg') else None) + if pos_or_neg == '+': + from_beginning = True + elif pos_or_neg == '-': + from_beginning = False + else: + from_beginning = pos_or_neg + size = int(pos.group('size')) + amt_type = pos.group('pct_unit_or_sct').strip() + else: + raise ValueError('Invalid size specified: {0}'.format(orig_pos)) + return((from_beginning, size, amt_type)) + + +class Partition(object): + def __init__(self, disk_xml, diskobj, start_sector): + self.xml = disk_xml + device = diskobj.device + sizes = {} + for s in ('start', 'stop'): + x = dict(zip(('from_bgn', 'size', 'type'), + convertSizeUnit(self.xml.attrib[s]))) + sectors = x['size'] + if x['type'] == '%': + sectors = int(device.getLength() / x['size']) + elif x['type'] in _units.keys(): + sectors = int(x['size'] << _units[x['type']] / device.sectorSize) + sizes[s] = (sectors, x['from_bgn']) + if sizes['start'][1] is not None: + if sizes['start'][1]: + self.begin = sizes['start'][0] + 0 + else: + self.begin = device.getLength() - sizes['start'][0] + else: + self.begin = sizes['start'][0] + start_sector + if sizes['stop'][1] is not None: + if sizes['stop'][1]: + self.end = sizes['stop'][0] + 0 + else: + self.end = device.getLength() - sizes['stop'][0] + else: + self.end = self.begin + sizes['stop'][0] + # TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit + # in configs. So we manually crunch the numbers and do it all at the end. + self.geometry = parted.Geometry(device = device, + start = self.begin, + end = self.end) + self.filesystem = parted.FileSystem(type = self.xml.attrib['fsType'], + ) + self.partition = parted.Partition(disk = diskobj, + type = parted.PARTITION_NORMAL, + geometry = self.geometry, + fs = ) + +class Disk(object): + def __init__(self, disk_xml): + self.xml = disk_xml + self.devpath = self.xml.attrib['device'] + self._initDisk() + + def _initDisk(self): + self.device = parted.getDevice(self.devpath) + try: + self.disk = parted.newDisk(self.device) + if xmlBool(self.xml.attrib.get('forceReformat')): + self.is_lowformatted = False + self.is_hiformatted = False + else: + self.is_lowformatted = True + self.is_hiformatted = False + for d in blkinfo.BlkDiskInfo().get_disks(filters = {'group': 'disk', + 'name': os.path.basename(self.devpath), + 'kname': os.path.basename(self.devpath)}): + if d.get('fstype', '').strip() != '': + self.is_hiformatted = True + break + self.is_partitioned = True + except parted._ped.DiskException: + self.disk = None + self.is_lowformatted = False + self.is_hiformatted = False + self.is_partitioned = False + return() + + def diskformat(self): + if self.is_lowformatted: + return() + # This is a safeguard. We do *not* want to low-format a disk that is mounted. + for p in psutil.disk_partitions(all = True): + if self.devpath in p: + raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) + self.disk.deleteAllPartitions() + tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower() + if tabletype in ('bios', 'mbr'): + tabletype = 'msdos' + validlabels = parted.getLabels() + if tabletype not in validlabels: + raise ValueError(('Disk format {0} is not valid for this architecture;' + 'must be one of: {1}'.format(tabletype, ', '.join(list(validlabels))))) + self.disk = parted.freshDisk(self.device, tabletype) + + pass + self.is_lowformatted = True + self.is_partitioned = True + return() + + def fsformat(self): + if self.is_hiformatted: + return() + # This is a safeguard. We do *not* want to high-format a disk that is mounted. + for p in psutil.disk_partitions(all = True): + if self.devpath in p: + raise RuntimeError('{0} is mounted; we are cowardly refusing to high-format it'.format(self.devpath)) + + pass + return() diff --git a/aif/envsetup.py b/aif/envsetup.py index 143b315..f59e846 100644 --- a/aif/envsetup.py +++ b/aif/envsetup.py @@ -7,13 +7,14 @@ import ensurepip import json import os -import tempfile import subprocess import sys +import tempfile import venv # TODO: a more consistent way of managing deps? -depmods = ['gpg', 'requests', 'lxml', 'psutil', 'pyparted', 'pytz', 'passlib', 'validators'] +depmods = ['blkinfo', 'gpg', 'lxml', 'passlib', 'psutil', + 'pyparted', 'pytz', 'requests', 'validators'] class EnvBuilder(object): def __init__(self): diff --git a/docs/README.adoc b/docs/README.adoc index a2e522a..3c9bfee 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -205,7 +205,7 @@ The `/aif/storage/disk` element holds information about disks on the system, and |====================== ^|Attribute ^|Value ^m|device |The disk to format (e.g. `/dev/sda`) -^m|diskfmt |https://en.wikipedia.org/wiki/GUID_Partition_Table[`gpt`^] or https://en.wikipedia.org/wiki/Master_boot_record[`bios`^] +^m|diskfmt |https://en.wikipedia.org/wiki/GUID_Partition_Table[`gpt`^] or https://en.wikipedia.org/wiki/Master_boot_record[`msdos`^] |====================== ===== `` @@ -223,10 +223,11 @@ The `/aif/storage/disk/part` element holds information on partitioning that it's [[specialsize]] The `start` and `stop` attributes can be in the form of: -* A percentage, indicated by a percentage sign (`"10%"`) -* A size, indicated by the abbreviation (`"300K"`, `"30G"`, etc.) -** Accepts *K* (Kilobytes), *M* (Megabytes), *G* (Gigabytes), *T* (Terabytes), or *P* (Petabytes -- I know, I know.) -** Can also accept modifiers for this form (`"+500G"`, `"-400M"`) +* A percentage of the total disk size, indicated by a percentage sign (`"10%"`) +* A size, indicated by the abbreviation (`"300KiB"`, `"10GB"`, etc.) +** Accepts notation in https://en.wikipedia.org/wiki/Binary_prefix[SI or IEC formats^] +* A raw sector size, if no suffix is provided (sector sizes are *typically* 512 bytes but this can vary depending on disk) (`1024`) +* One can also specify modifiers (`"+10%"`, `"-400MB"`, etc.). A positive modifier indicates from the beginning of the *start of the disk* and a negative modifier specifies from the *end of the disk* (the default, if none is specified, is to use the _previously defined partition's end_ as the *start* for the new partition, or to use the _beginning of the usable disk space_ as the *start* if no previous partition is specified, and to *add* the size to the *start* until the *stop* is reached) [[fstypes]] NOTE: The following is a table for your reference of partition types. Note that it may be out of date, so reference the link above for the most up-to-date table. diff --git a/docs/TODO b/docs/TODO index 6c766e5..3dab1a9 100644 --- a/docs/TODO +++ b/docs/TODO @@ -1,3 +1,4 @@ +- make disk partitioning/table formatting OPTIONAL (so it can be installed on an already formatted disk) - support Arch Linux ARM? - support multiple explicit locales via comma-separated list (see how i handle resolvers) - config layout @@ -25,6 +26,7 @@ shm on /mnt/aif/dev/shm type tmpfs (rw,nosuid,nodev,relatime) run on /mnt/aif/run type tmpfs (rw,nosuid,nodev,relatime,mode=755) tmp on /mnt/aif/tmp type tmpfs (rw,nosuid,nodev) + OR just use pyalpm DOCUMENTATION: aif-config.py (and note sample json as well) diff --git a/docs/examples/aif.xml b/docs/examples/aif.xml index c2c83f3..602e5dc 100644 --- a/docs/examples/aif.xml +++ b/docs/examples/aif.xml @@ -1,54 +1,77 @@ - + - - - - + + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + - + - + - + - - - + + + - - - - - - - + + + + + + + - - http://mirrors.advancedhosters.com/archlinux/$repo/os/$arch - http://mirrors.advancedhosters.com/archlinux/$repo/os/$arch + + http://arch.mirror.square-r00t.net/$repo/os/$arch http://mirror.us.leaseweb.net/archlinux/$repo/os/$arch http://ftp.osuosl.org/pub/archlinux/$repo/os/$arch http://arch.mirrors.ionfish.org/$repo/os/$arch http://mirrors.gigenet.com/archlinux/$repo/os/$arch http://mirror.jmu.edu/pub/archlinux/$repo/os/$arch - + diff --git a/setup.py b/setup.py index ce63f76..61c7197 100644 --- a/setup.py +++ b/setup.py @@ -31,5 +31,6 @@ setuptools.setup( project_urls = {'Documentation': 'https://aif-ng.io/', 'Source': 'https://git.square-r00t.net/AIF-NG/', 'Tracker': 'https://bugs.square-r00t.net/index.php?project=9'}, - install_requires = ['gpg', 'requests', 'lxml', 'psutil', 'pyparted', 'pytz', 'passlib', 'validators'] + install_requires = ['blkinfo', 'gpg', 'lxml', 'passlib', 'psutil', + 'pyparted', 'pytz', 'requests', 'validators'] )