cleaned up, etc.

2019-10-30 03:46:33 -04:00 · 2019-10-30 03:46:33 -04:00 · c4386d55d1
parent f0d93658d0
commit c4386d55d1
10 changed files with 33 additions and 396 deletions
--- a/aif/constants.py
+++ b/aif/constants.py
@ -1,3 +1,12 @@
-import os
-import re
-
+version = '0.2.0'
+external_deps = ['blkinfo',
+                 'gpg',
+                 'lxml',
+                 'mdstat',
+                 'passlib',
+                 'psutil',
+                 'pyparted',
+                 'pyroute2',
+                 'pytz',
+                 'requests',
+                 'validators']
--- a/aif/disk/block.py
+++ b/aif/disk/block.py
@ -18,7 +18,7 @@ import blkinfo
 import parted  # https://www.gnu.org/software/parted/api/index.html
 import psutil
 ##
-from aif.utils import xmlBool
+from aif.utils import xmlBool, size


 PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys()))
@ -26,23 +26,11 @@ PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values()))
 IDX_FLAG = dict(parted.partition.partitionFlag)
 FLAG_IDX = {v: k for k, v in IDX_FLAG.items()}

-# parted lib can do SI or IEC (see table to right at https://en.wikipedia.org/wiki/Binary_prefix)
-# We bit-shift to do conversions:
-# https://stackoverflow.com/a/12912296/733214
-# https://stackoverflow.com/a/52684562/733214
-_units = {'B': 0,
-          'kB': 7,
-          'MB': 17,
-          'GB': 27,
-          'TB': 37,
-          'KiB': 10,
-          'MiB': 20,
-          'GiB': 30,
-          'TiB': 40}
+# parted lib can do SI or IEC. So can we.
 _pos_re = re.compile((r'^(?P<pos_or_neg>-|\+)?\s*'
                      r'(?P<size>[0-9]+)\s*'
                      # empty means size in sectors
-                      r'(?P<pct_unit_or_sct>%|{0}|)\s*$'.format('|'.join(list(_units.keys())))
+                      r'(?P<pct_unit_or_sct>%|{0}|)\s*$'.format('|'.join(size.valid_storage))
                      ))


@ -57,11 +45,11 @@ def convertSizeUnit(pos):
            from_beginning = False
        else:
            from_beginning = pos_or_neg
-        size = int(pos.group('size'))
+        _size = int(pos.group('size'))
        amt_type = pos.group('pct_unit_or_sct').strip()
    else:
        raise ValueError('Invalid size specified: {0}'.format(orig_pos))
-    return((from_beginning, size, amt_type))
+    return((from_beginning, _size, amt_type))


 class Partition(object):
@ -105,8 +93,8 @@ class Partition(object):
            sectors = x['size']
            if x['type'] == '%':
                sectors = int(self.device.getLength() / x['size'])
-            elif x['type'] in _units.keys():
-                sectors = int(x['size'] << _units[x['type']] / self.device.sectorSize)
+            else:
+                sectors = int(size.convertStorage(x['size'], x['type'], target = 'B') / self.device.sectorSize)
            sizes[s] = (sectors, x['from_bgn'])
        if sizes['start'][1] is not None:
            if sizes['start'][1]:
--- a/aif/disk/filesystem.py
+++ b/aif/disk/filesystem.py
@ -36,8 +36,9 @@ for i in os.listdir(_mod_dir):
    if fs_name:
        # The kernel *probably* has autoloading enabled, but in case it doesn't...
        # TODO: logging!
-        subprocess.run(['modprobe', fs_name])
-        FS_FSTYPES.append(fs_name)
+        if os.getuid() == 0:
+            subprocess.run(['modprobe', fs_name])
+            FS_FSTYPES.append(fs_name)


 class FS(object):
--- a/aif/disk/mdadm.py
+++ b/aif/disk/mdadm.py
@ -11,7 +11,7 @@ from aif.disk.block import Disk
 from aif.disk.block import Partition


-SUPPORTED_LEVELS = (0, 1, 4, 5, 6)
+SUPPORTED_LEVELS = (0, 1, 4, 5, 6, 10)
 SUPPORTED_METADATA = ('0', '0.90', '1', '1.0', '1.1', '1.2', 'default', 'ddf', 'imsm')
 SUPPORTED_LAYOUTS = {5: (re.compile(r'^((left|right)-a?symmetric|[lr][as]|'
                                    r'parity-(fir|la)st|'
--- a/aif/envsetup.py
+++ b/aif/envsetup.py
@ -11,10 +11,8 @@ import subprocess
 import sys
 import tempfile
 import venv
-
-# TODO: a more consistent way of managing deps?
-depmods = ['blkinfo', 'gpg', 'lxml', 'mdstat', 'passlib', 'psutil',
-           'pyparted', 'pyroute2', 'pytz', 'requests', 'validators']
+##
+import aif.constants

 class EnvBuilder(object):
    def __init__(self):
@ -37,7 +35,7 @@ class EnvBuilder(object):
        # This is SO. DUMB. WHY DO I HAVE TO CALL PIP FROM A SHELL. IT'S WRITTEN IN PYTHON.
        # https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
        # TODO: logging
-        for m in depmods:
+        for m in aif.constants.external_deps:
            pip_cmd = [os.path.join(self.vdir,
                                    'bin',
                                    'python3'),
--- a/aif/utils.py
+++ b/aif/utils.py
@ -38,6 +38,8 @@ def collapseValues(d, vallist = None):
 class _Sizer(object):
    def __init__(self):
        # We use different methods for converting between storage and BW, and different multipliers for each subtype.
+        # https://stackoverflow.com/a/12912296/733214
+        # https://stackoverflow.com/a/52684562/733214
        # https://stackoverflow.com/questions/5194057/better-way-to-convert-file-sizes-in-python
        # https://en.wikipedia.org/wiki/Orders_of_magnitude_(data)
        # https://en.wikipedia.org/wiki/Binary_prefix
@ -92,13 +94,13 @@ class _Sizer(object):
        for unit_type, convpair in self.storageUnits.items():
            for f, l in convpair.items():
                for suffix in l:
-                    if suffix not in self.valid_storage:
+                    if suffix not in self.valid_storage and suffix:
                        self.valid_storage.append(suffix)
        self.valid_bw = []
        for unit_type, convpair in self.bwUnits.items():
            for f, l in convpair.items():
                for suffix in l:
-                    if suffix not in self.valid_bw:
+                    if suffix not in self.valid_bw and suffix:
                        self.valid_bw.append(suffix)

    def convert(self, n, suffix):
--- a/extras/bdisk.build.ini
+++ b/extras/bdisk.build.ini
@ -1,104 +0,0 @@
-###########################################################
-##                BUILD.CONF SAMPLE FILE                 ##
-###########################################################
-#
-# This file is used to define various variables/settings
-# used by the build script.
-#
-# For full (perhaps overly-verbose ;) documentation, please
-# see:
-# https://bdisk.square-r00t.net/#_the_code_build_ini_code_file
-# Or simply refer to the section titled "The build.ini File"
-# in the user manual.
-
-[bdisk]
-name = AIF
-uxname = aif
-pname = AIF-NG
-ver = 1.00
-dev = r00t^2
-email = bts@square-r00t.net
-desc = See https://aif.square-r00t.net/
-uri = https://aif.square-r00t.net/
-root_password = BLANK
-user = no
-
-[user]
-username = ${bdisk:uxname}
-name = Default user
-password = BLANK
-
-[source_x86_64]
-mirror = mirror.us.leaseweb.net
-#mirrorproto = https
-mirrorproto = http
-mirrorpath = /archlinux/iso/latest/
-mirrorfile = 
-mirrorchksum = ${mirrorpath}sha1sums.txt
-chksumtype = sha1
-mirrorgpgsig = .sig
-gpgkey = 4AA4767BBC9C4B1D18AE28B77F2D434B9741E8AC
-gpgkeyserver =
-
-[source_i686]
-mirror = mirror.us.leaseweb.net
-#mirrorproto = https
-mirrorproto = http
-mirrorpath = /archlinux/iso/latest/
-mirrorfile = 
-mirrorchksum = ${mirrorpath}sha1sums.txt
-chksumtype = sha1
-mirrorgpgsig = .sig
-gpgkey = 7F2D434B9741E8AC
-gpgkeyserver =
-
-[build]
-gpg = yes
-dlpath = /var/tmp/${bdisk:uxname}
-chrootdir = /var/tmp/chroots
-basedir = /opt/dev/bdisk
-isodir = ${dlpath}/iso
-srcdir = ${dlpath}/src
-prepdir = ${dlpath}/temp
-archboot = ${prepdir}/${bdisk:name}
-mountpt = /mnt/${bdisk:uxname}
-multiarch = 64
-sign = yes
-ipxe = yes
-i_am_a_racecar = yes
-
-[gpg]
-mygpgkey = 748231EBCBD808A14F5E85D28C004C2F93481F6B
-mygpghome = /root/.gnupg
-
-[sync]
-http = yes
-tftp = yes
-git = no
-rsync = no
-
-[http]
-path = ${build:dlpath}/http
-user = root
-group = root
-
-[tftp]
-path = ${build:dlpath}/tftpboot
-user = root
-group = root
-
-[ipxe]
-iso = yes
-uri = https://aif.square-r00t.net/boot.ipxe
-ssldir = ${build:dlpath}/ssl
-ssl_ca = ${ssldir}/ca.crt
-ssl_cakey = ${ssldir}/ca.key
-ssl_crt = ${ssldir}/main.crt
-ssl_key = ${ssldir}/main.key
-
-[rsync]
-#host = 10.1.1.1
-host = bdisk.square-r00t.net
-user = root
-path = /srv/http/bdisk_ipxe
-iso = yes
--- a/extras/createtest.expect
+++ b/extras/createtest.expect
@ -1,208 +0,0 @@
-#!/usr/bin/expect -f
-
-log_file -noappend /tmp/expect.log
-set force_conservative 0  ;# set to 1 to force conservative mode even if
-			  ;# script wasn't run conservatively originally
-if {$force_conservative} {
-	set send_slow {1 .1}
-	proc send {ignore arg} {
-		sleep .1
-		exp_send -s -- $arg
-	}
-}
-
-#set send_slow {10 .001}
-
-set timeout -1
-#spawn ./aif-config.py create -v:r -f /tmp/aif.xml
-spawn ./aif-config.py create -v -f /tmp/aif.xml
-## disks
-send -- "/dev/sda,/dev/sdb\r"
-# sda
-send -- "gpt\r"
-send -- "2\r"
-# sda1
-send -- "0%\r"
-send -- "95%\r"
-send -- "8300\r"
-# sda2
-send -- "95%\r"
-send -- "100%\r"
-send -- "ef00\r"
-# sdb
-send -- "gpt\r"
-send -- "3\r"
-# sdb1
-send -- "0%\r"
-send -- "47%\r"
-send -- "8300\r"
-# sdb2
-send -- "47%\r"
-send -- "95%\r"
-send -- "8300\r"
-# sdb3
-send -- "95%\r"
-send -- "100%\r"
-send -- "8200\r"
-## mounts
-send -- "/mnt/aif,/mnt/aif/boot,/mnt/aif/home,/mnt/aif/mnt/data,swap\r"
-# /mnt/aif
-send -- "/dev/sda1\r"
-send -- "1\r"
-send -- "ext4\r"
-send -- "defaults\r"
-# /mnt/aif/boot
-send -- "/dev/sda2\r"
-send -- "2\r"
-send -- "vfat\r"
-send -- "rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro\r"
-# /mnt/aif/home
-send -- "/dev/sdb1\r"
-send -- "3\r"
-send -- "ext4\r"
-send -- "defaults\r"
-# /mnt/aif/mnt/data
-send -- "/dev/sdb2\r"
-send -- "4\r"
-send -- "ext4\r"
-send -- "defaults\r"
-# swap
-send -- "/dev/sdb3\r"
-send -- "5\r"
-## network
-# hostname
-send -- "aif.loc.lan\r"
-# interface
-send -- "ens3\r"
-send -- "auto\r"
-send -- "ipv4\r"
-# add another interface?
-send -- "y\r"
-# second interface
-send -- "ens4\r"
-send -- "192.168.1.2/24\r"
-send -- "192.168.1.1\r"
-send -- "4.2.2.1,4.2.2.2\r"
-# add another interface? default is no
-send -- "\r"
-## system
-# timezone (default is UTC)
-send -- "\r"
-# locale (default is en_US.UTF-8
-send -- "\r"
-# chroot path
-send -- "/mnt/aif\r"
-# kbd (default is US)
-send -- "\r"
-# reboot host after install? default is yes
-send -- "\r"
-# root password
-sleep 2
-send -- "test\r"
-sleep 2
-expect *
-# add user?
-send -- "y\r"
-# user
-send -- "aifusr\r"
-# sudo access
-send -- "y\r"
-# password
-sleep 2
-send -- "test\r"
-sleep 2
-send -- "A Test User\r"
-# uid (default is autogen)
-send -- "\r"
-# primary group (default is autogen'd based on username)
-send -- "\r"
-# home dir (default is e.g. /home/username)
-send -- "\r"
-# add exta groups?
-send -- "y\r"
-# extra group
-send -- "users\r"
-# need to be created? default is no
-send -- "\r"
-# add another extra group? default is no
-send -- "\r"
-# add more users? default is no
-send -- "\r"
-# enable/disable services
-send -- "y\r"
-# service
-send -- "sshd\r"
-# enable? default is yes
-send -- "\r"
-# manage another service? default is no
-send -- "\r"
-# packager (default is pacman)
-send -- "\r"
-# review default repos? default is yes
-send -- "\r"
-# edit any of them?
-send -- "y\r"
-# edit the 6th repo (multilib)
-send -- "6\r"
-# enabled?
-send -- "y\r"
-# siglevel (default is unchanged)
-send -- "\r"
-# mirror URI (default is unchanged)
-send -- "\r"
-# edit another repo? default is no
-send -- "\r"
-# add additional repositories? default is no
-send -- "\r"
-# modify default mirrorlist?
-send -- "y\r"
-# URI for mirror
-send -- "http://mirrors.advancedhosters.com/archlinux/\$repo/os/\$arch\r"
-# add another?
-send -- "y\r"
-send -- "http://mirror.us.leaseweb.net/archlinux/\$repo/os/\$arch\r"
-send -- "y\r"
-send -- "http://arch.mirror.constant.com/\$repo/os/\$arch\r"
-send -- "y\r"
-send -- "http://mirror.vtti.vt.edu/archlinux/\$repo/os/\$arch\r"
-send -- "y\r"
-send -- "http://arch.mirrors.pair.com/\$repo/os/\$arch\r"
-send -- "y\r"
-send -- "http://mirror.yellowfiber.net/archlinux/\$repo/os/\$arch\r"
-send -- "\r"
-# install extra software?
-send -- "y\r"
-# software
-send -- "openssh\r"
-# repository (optional)
-send -- "\r"
-# add another package?
-send -- "\r"
-# bootloader (default is grub)
-send -- "\r"
-# system supports UEFI? default is yes
-send -- "\r"
-# ESP/EFI system partition
-send -- "/boot\r"
-# any hook scripts? default is no
-send -- "y\r"
-# pre, pkg, or post
-send -- "post\r"
-# script URI
-send -- "https://aif.square-r00t.net/sample-scripts/post/first.sh\r"
-# order for the execution run
-send -- "1\r"
-# auth required?
-send -- "y\r"
-# basic/digest? default is basic
-send -- "digest\r"
-# if digest, realm
-send -- "realmname\r"
-# user
-send -- "test\r"
-# password
-send -- "password\r"
-# would you like to add another script? default is no
-send -- "\r"
-interact
-expect eof
--- a/extras/txttojson.py
+++ b/extras/txttojson.py
@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import json
-import os
-import pprint
-#import re
-try:
-    import yaml
-except:
-    exit('You need pyYAML.')
-
-def parseArgs():
-    args = argparse.ArgumentParser()
-    args.add_argument('-i',
-                      '--in',
-                      dest = 'infile',
-                      required = True,
-                      help = 'The plaintext representation of a python dict')
-    args.add_argument('-o',
-                      '--out',
-                      dest = 'outfile',
-                      required = True,
-                      help = 'The JSON file to create')
-    return(args)
-
-def main():
-    args = vars(parseArgs().parse_args())
-    infile = os.path.abspath(os.path.normpath(args['infile']))
-    outfile = os.path.abspath(os.path.normpath(args['outfile']))
-    if not os.path.lexists(infile):
-        exit('Input file doesn\'t exist.')
-#try:
-    with open(outfile, 'w') as outgoing:
-        with open(infile, 'r') as incoming:
-            #data = re.sub("'", '"', incoming.read())
-            #outgoing.write(data)
-            #d = json.dumps(data, ensure_ascii = False)
-            #d = json.dumps(incoming.read().replace("'", '"'))
-            d = yaml.load(incoming.read())
-            pprint.pprint(d)
-            j = json.dumps(d, indent = 4)
-            outgoing.write(j)
-#except:
-    #exit('Error when trying to read/write file(s).')
-    return()
-
-if __name__ == '__main__':
-    main()
--- a/setup.py
+++ b/setup.py
@ -1,11 +1,12 @@
 import setuptools
+import aif.constants as PROJ_CONST

 with open('README', 'r') as fh:
    long_description = fh.read()

 setuptools.setup(
        name = 'aif',
-        version = '0.2.0',
+        version = PROJ_CONST.version,
        author = 'Brent S.',
        author_email = 'bts@square-r00t.net',
        description = 'Arch Installation Framework (Next Generation)',
@ -31,6 +32,5 @@ setuptools.setup(
        project_urls = {'Documentation': 'https://aif-ng.io/',
                        'Source': 'https://git.square-r00t.net/AIF-NG/',
                        'Tracker': 'https://bugs.square-r00t.net/index.php?project=9'},
-        install_requires = ['blkinfo', 'gpg', 'lxml', 'mdstat', 'passlib', 'psutil',
-                            'pyparted', 'pyroute2', 'pytz', 'requests', 'validators']
+        install_requires = PROJ_CONST.external_deps
        )