From c22b473b495cd2762fae79f171441b8b93f86578 Mon Sep 17 00:00:00 2001 From: brent s Date: Mon, 30 Dec 2019 12:59:52 -0500 Subject: [PATCH] checking in before i change a regex pattern. this currently will grab commented out defaults, but we don't want that since it complicates things - so we hardcode in shipped defaults. --- aif/disk/main.py | 1 + aif/envsetup.py | 36 +++++++++++++++---- aif/network/netctl.py | 4 +-- aif/network/networkmanager.py | 2 +- aif/pacman.py | 6 ---- aif/pacman/__init__.py | 68 +++++++++++++++++++++++++++++++++++ aif/pacman/_common.py | 19 ++++++++++ aif/system/console.py | 4 +-- aif/system/locales.py | 2 +- 9 files changed, 124 insertions(+), 18 deletions(-) delete mode 100644 aif/pacman.py create mode 100644 aif/pacman/__init__.py create mode 100644 aif/pacman/_common.py diff --git a/aif/disk/main.py b/aif/disk/main.py index 4640904..fe61cb5 100644 --- a/aif/disk/main.py +++ b/aif/disk/main.py @@ -1 +1,2 @@ # TODO +# Remember to genfstab! diff --git a/aif/envsetup.py b/aif/envsetup.py index cfe5a86..b419c00 100644 --- a/aif/envsetup.py +++ b/aif/envsetup.py @@ -1,11 +1,13 @@ -# We use a temporary venv to ensure we have all the external libraries we need. -# This removes the necessity of extra libs at runtime. If you're in an environment that doesn't have access to PyPI/pip, -# you'll need to customize the install host (typically the live CD/live USB) to have them installed as system packages. +# This can set up an environment at runtime. +# This removes the necessity of extra libs to be installed persistently. +# However, it is recommended that you install all dependencies in the system itself, because some aren't available +# through pip/PyPi. # Before you hoot and holler about this, Let's Encrypt's certbot-auto does the same thing. # Except I segregate it out even further; I don't even install pip into the system python. import ensurepip import json +import logging import os import subprocess import sys @@ -14,6 +16,10 @@ import venv ## import aif.constants_fallback + +_logger = logging.getLogger(__name__) + + class EnvBuilder(object): def __init__(self): self.vdir = tempfile.mkdtemp(prefix = '.aif_', suffix = '_VENV') @@ -30,11 +36,20 @@ class EnvBuilder(object): ('import site; ' 'import json; ' 'print(json.dumps(site.getsitepackages(), indent = 4))')], - stdout = subprocess.PIPE) + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + _logger.info('Executed: {0}'.format(' '.join(moddir_raw.args))) + if moddir_raw.returncode != 0: + _logger.warning('Command returned non-zero status') + _logger.debug('Exit status: {0}'.format(str(moddir_raw.returncode))) + for a in ('stdout', 'stderr'): + x = getattr(moddir_raw, a) + if x: + _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) + raise RuntimeError('Failed to establish environment successfully') self.modulesdir = json.loads(moddir_raw.stdout.decode('utf-8'))[0] # This is SO. DUMB. WHY DO I HAVE TO CALL PIP FROM A SHELL. IT'S WRITTEN IN PYTHON. # https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program - # TODO: logging for m in aif.constants_fallback.EXTERNAL_DEPS: pip_cmd = [os.path.join(self.vdir, 'bin', @@ -44,6 +59,15 @@ class EnvBuilder(object): 'install', '--disable-pip-version-check', m] - subprocess.run(pip_cmd) + cmd = subprocess.run(pip_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + _logger.info('Executed: {0}'.format(' '.join(cmd.args))) + if cmd.returncode != 0: + _logger.warning('Command returned non-zero status') + _logger.debug('Exit status: {0}'.format(str(cmd.returncode))) + for a in ('stdout', 'stderr'): + x = getattr(cmd, a) + if x: + _logger.debug('{0}: {1}'.format(a.upper(), x.decode('utf-8').strip())) + raise RuntimeError('Failed to install module successfully') # And now make it available to other components. sys.path.insert(1, self.modulesdir) diff --git a/aif/network/netctl.py b/aif/network/netctl.py index 4f735a6..cf7f218 100644 --- a/aif/network/netctl.py +++ b/aif/network/netctl.py @@ -37,7 +37,7 @@ class Connection(_common.BaseConnection): self.device = _common.getDefIface(self.connection_type) self.desc = ('A {0} profile for {1} (generated by AIF-NG)').format(self.connection_type, self.device) - self._cfg = configparser.ConfigParser() + self._cfg = configparser.ConfigParser(allow_no_value = True, interpolation = None) self._cfg.optionxform = str # configparser *requires* sections. netctl doesn't use them. We strip it when we write. self._cfg['BASE'] = {'Description': self.desc, @@ -130,7 +130,7 @@ class Connection(_common.BaseConnection): fulld = os.path.join(root, d) os.chmod(fulld, 0o0755) os.chown(fulld, 0, 0) - systemd_cfg = configparser.ConfigParser() + systemd_cfg = configparser.ConfigParser(allow_no_value = True, interpolation = None) systemd_cfg.optionxform = str systemd_cfg['Unit'] = {'Description': self.desc, 'BindsTo': 'sys-subsystem-net-devices-{0}.device'.format(self.device), diff --git a/aif/network/networkmanager.py b/aif/network/networkmanager.py index bb6113a..cd915f9 100644 --- a/aif/network/networkmanager.py +++ b/aif/network/networkmanager.py @@ -32,7 +32,7 @@ class Connection(_common.BaseConnection): _logger.info('Building config.') if self.device == 'auto': self.device = _common.getDefIface(self.connection_type) - self._cfg = configparser.ConfigParser() + self._cfg = configparser.ConfigParser(allow_no_value = True, interpolation = None) self._cfg.optionxform = str self._cfg['connection'] = {'id': self.id, 'uuid': self.uuid, diff --git a/aif/pacman.py b/aif/pacman.py deleted file mode 100644 index 4d0270c..0000000 --- a/aif/pacman.py +++ /dev/null @@ -1,6 +0,0 @@ -# We can manually bootstrap and alter pacman's keyring. But check the bootstrap tarball; we might not need to. -# TODO. - -import os -## -import gpg diff --git a/aif/pacman/__init__.py b/aif/pacman/__init__.py new file mode 100644 index 0000000..b38ed97 --- /dev/null +++ b/aif/pacman/__init__.py @@ -0,0 +1,68 @@ +# We can manually bootstrap and alter pacman's keyring. But check the bootstrap tarball; we might not need to. +# TODO. + +import configparser +import logging +import os +import re +## +import pyalpm +import gpg +## +from . import _common + + +_logger = logging.getLogger(__name__) + + +_skipconfline_re = re.compile(r'^[ \t]*#([ \t]+.*)?$') + + +class PackageManager(object): + def __init__(self, chroot_base): + self.chroot_base = chroot_base + self.pacman_dir = os.path.join(self.chroot_base, 'var', 'lib', 'pacman') + self.configfile = os.path.join(self.chroot_base, 'etc', 'pacman.conf') + self.config = None + self._parseConf() + + def _parseConf(self): + _cf = [] + with open(self.configfile, 'r') as fh: + for line in fh.read().splitlines(): + if _skipconfline_re.search(line) or line.strip() == '': + continue + _cf.append(re.sub(r'^#', '', line)) + self.config = configparser.ConfigParser(allow_no_value = True, + interpolation = None, + strict = False, + dict_type = _common.MultiOrderedDict) + self.config.optionxform = str + self.config.read_string('\n'.join(_cf)) + self.opts = {'Architecture': 'auto', + 'CacheDir': '/var/cache/pacman/pkg/', + 'CheckSpace': None, + 'CleanMethod': 'KeepInstalled', + # 'Color': None, + 'DBPath': '/var/lib/pacman/', + 'GPGDir': '/etc/pacman.d/gnupg/', + 'HoldPkg': 'pacman glibc', + 'HookDir': '/etc/pacman.d/hooks/', + 'IgnoreGroup': '', + 'IgnorePkg': '', + 'LocalFileSigLevel': 'Optional', + 'LogFile': '/var/log/pacman.log', + 'NoExtract': '', + 'NoUpgrade': '', + 'RemoteFileSigLevel': 'Required', + 'RootDir': '/', + 'SigLevel': 'Required DatabaseOptional', + # 'TotalDownload': None, + # 'UseSyslog': None, + # 'VerbosePkgLists': None, + 'XferCommand': '/usr/bin/curl -L -C - -f -o %o %u' + } + self.distro_repos = [''] + _opts = dict(self.config.items('options')) + self.opts.update(_opts) + self.config.remove_section('options') diff --git a/aif/pacman/_common.py b/aif/pacman/_common.py new file mode 100644 index 0000000..d51d615 --- /dev/null +++ b/aif/pacman/_common.py @@ -0,0 +1,19 @@ +import configparser +import logging +from collections import OrderedDict + + +_logger = logging.getLogger('pacman:_common') + + +class MultiOrderedDict(OrderedDict): + # Thanks, dude: https://stackoverflow.com/a/38286559/733214 + def __setitem__(self, key, value): + if key in self: + if isinstance(value, list): + self[key].extend(value) + return(None) + elif isinstance(value, str): + if len(self[key]) > 1: + return(None) + super(MultiOrderedDict, self).__setitem__(key, value) diff --git a/aif/system/console.py b/aif/system/console.py index d707544..4a74053 100644 --- a/aif/system/console.py +++ b/aif/system/console.py @@ -17,8 +17,8 @@ class Console(object): def __init__(self, chroot_base, console_xml): self.xml = console_xml self.chroot_base = chroot_base - self._cfg = configparser.ConfigParser() - self._cfg.optionxform(str) + self._cfg = configparser.ConfigParser(allow_no_value = True, interpolation = None) + self._cfg.optionxform = str self.keyboard = Keyboard(self.xml.find('keyboard')) self.font = Font(self.xml.find('text')) self._cfg['BASE'] = {} diff --git a/aif/system/locales.py b/aif/system/locales.py index 460e391..04fad9d 100644 --- a/aif/system/locales.py +++ b/aif/system/locales.py @@ -23,7 +23,7 @@ class Locale(object): self.syslocales = {} self.userlocales = [] self.rawlocales = None - self._localevars = configparser.ConfigParser() + self._localevars = configparser.ConfigParser(allow_no_value = True, interpolation = None) self._localevars.optionxform = str self._localevars['BASE'] = {} self._initVars()