From 7401fed6d3c3739308ac34a7ce3b809991264505 Mon Sep 17 00:00:00 2001 From: brent s Date: Fri, 10 Jan 2020 08:39:28 -0500 Subject: [PATCH] updating some pacman stuff. need to finish objtypes.Repo and might need to tweak config writer. --- aif/__init__.py | 2 +- aif/pacman/_common.py | 22 ---- aif/software/__init__.py | 4 + aif/software/config.py | 124 ++++++++++++++++++ aif/{pacman => software}/keyring.py | 2 + aif/software/mirrorlist.j2 | 5 + aif/software/objtypes.py | 72 ++++++++++ aif/software/pacman.conf.j2 | 16 +++ .../__init__.py => software/pacman.py} | 70 +--------- 9 files changed, 227 insertions(+), 90 deletions(-) delete mode 100644 aif/pacman/_common.py create mode 100644 aif/software/__init__.py create mode 100644 aif/software/config.py rename aif/{pacman => software}/keyring.py (99%) create mode 100644 aif/software/mirrorlist.j2 create mode 100644 aif/software/objtypes.py create mode 100644 aif/software/pacman.conf.j2 rename aif/{pacman/__init__.py => software/pacman.py} (64%) diff --git a/aif/__init__.py b/aif/__init__.py index 3c222fa..f36b9d7 100644 --- a/aif/__init__.py +++ b/aif/__init__.py @@ -14,7 +14,7 @@ from . import system from . import config from . import envsetup from . import network -from . import pacman +from . import software _logger = logging.getLogger('AIF') diff --git a/aif/pacman/_common.py b/aif/pacman/_common.py deleted file mode 100644 index 10e3877..0000000 --- a/aif/pacman/_common.py +++ /dev/null @@ -1,22 +0,0 @@ -import configparser -import logging -from collections import OrderedDict - - -# TODO: Add pacman.conf parsing? - - -_logger = logging.getLogger('pacman:_common') - - -class MultiOrderedDict(OrderedDict): - # Thanks, dude: https://stackoverflow.com/a/38286559/733214 - def __setitem__(self, key, value): - if key in self: - if isinstance(value, list): - self[key].extend(value) - return(None) - elif isinstance(value, str): - if len(self[key]) > 1: - return(None) - super(MultiOrderedDict, self).__setitem__(key, value) diff --git a/aif/software/__init__.py b/aif/software/__init__.py new file mode 100644 index 0000000..975bdcb --- /dev/null +++ b/aif/software/__init__.py @@ -0,0 +1,4 @@ +from . import config +from . import keyring +from . import objtypes +from . import pacman diff --git a/aif/software/config.py b/aif/software/config.py new file mode 100644 index 0000000..08c343c --- /dev/null +++ b/aif/software/config.py @@ -0,0 +1,124 @@ +import copy +import logging +import os +import re +import shutil +from collections import OrderedDict +## +import jinja2 +## +import aif.utils + + +_logger = logging.getLogger(__name__) + + +class PacmanConfig(object): + _sct_re = re.compile(r'^\s*\[(?P[^]]+)\]\s*$') + _kv_re = re.compile(r'^\s*(?P[^\s=[]+)((?:\s*=\s*)(?P.*))?$') + _skipline_re = re.compile(r'^\s*(#.*)?$') + # TODO: Append mirrors/repos to pacman.conf here before we parse? + # I copy a log of logic from pycman/config.py here. + _list_keys = ('CacheDir', 'HookDir', 'HoldPkg', 'SyncFirst', 'IgnoreGroup', 'IgnorePkg', 'NoExtract', 'NoUpgrade', + 'Server') + _single_keys = ('RootDir', 'DBPath', 'GPGDir', 'LogFile', 'Architecture', 'XferCommand', 'CleanMethod', 'SigLevel', + 'LocalFileSigLevel', 'RemoteFileSigLevel') + _noval_keys = ('UseSyslog', 'ShowSize', 'TotalDownload', 'CheckSpace', 'VerbosePkgLists', 'ILoveCandy', 'Color', + 'DisableDownloadTimeout') + # These are the default (commented-out) values in the stock /etc/pacman.conf as of January 5, 2020. + defaults = OrderedDict({'options': {'Architecture': 'auto', + 'CacheDir': '/var/cache/pacman/pkg/', + 'CheckSpace': None, + 'CleanMethod': 'KeepInstalled', + # 'Color': None, + 'DBPath': '/var/lib/pacman/', + 'GPGDir': '/etc/pacman.d/gnupg/', + 'HoldPkg': 'pacman glibc', + 'HookDir': '/etc/pacman.d/hooks/', + 'IgnoreGroup': [], + 'IgnorePkg': [], + 'LocalFileSigLevel': ['Optional'], + 'LogFile': '/var/log/pacman.log', + 'NoExtract': [], + 'NoUpgrade': [], + 'RemoteFileSigLevel': ['Required'], + 'RootDir': '/', + 'SigLevel': ['Required', 'DatabaseOptional'], + # 'TotalDownload': None, + # 'UseSyslog': None, + # 'VerbosePkgLists': None, + 'XferCommand': '/usr/bin/curl -L -C - -f -o %o %u'}, + # These should be explicitly included in the AIF config. + # 'core': {'Include': '/etc/pacman.d/mirrorlist'}, + # 'extra': {'Include': '/etc/pacman.d/mirrorlist'}, + # 'community': {'Include': '/etc/pacman.d/mirrorlist'} + }) + + def __init__(self, chroot_base, confpath = '/etc/pacman.conf'): + self.chroot_base = chroot_base + self.confpath = os.path.join(self.chroot_base, re.sub(r'^/+', '', confpath)) + self.confbak = '{0}.bak'.format(self.confpath) + self.mirrorlstpath = os.path.join(self.chroot_base, 'etc', 'pacman.d', 'mirrorlist') + self.mirrorlstbak = '{0}.bak'.format(self.mirrorlstpath) + if not os.path.isfile(self.confbak): + shutil.copy2(self.confpath, self.confbak) + _logger.info('Copied: {0} => {1}'.format(self.confpath, self.confbak)) + if not os.path.isfile(self.mirrorlstbak): + shutil.copy2(self.mirrorlstpath, self.mirrorlstbak) + _logger.info('Copied: {0} => {1}'.format(self.mirrorlstpath, self.mirrorlstbak)) + self.j2_env = jinja2.Environment(loader = jinja2.FileSystemLoader(searchpath = './')) + self.j2_env.filters.update(aif.utils.j2_filters) + self.j2_conf = self.j2_env.get_template('pacman.conf.j2') + self.j2_mirror = self.j2_env.get_template('mirrorlist.j2') + self.conf = None + self.mirrors = [] + + def _includeExpander(self, lines): + curlines = [] + for line in lines: + r = self._kv_re.search(line) + if r and (r.group('key') == 'Include') and r.group('value'): + path = os.path.join(self.chroot_base, re.sub(r'^/?', '', r.group('path'))) + with open(path, 'r') as fh: + curlines.extend(self._includeExpander(fh.read().splitlines())) + else: + curlines.append(line) + return(curlines) + + def parse(self, defaults = True): + self.conf = OrderedDict() + rawlines = {} + with open(self.confpath, 'r') as fh: + rawlines['orig'] = [line for line in fh.read().splitlines() if not self._skipline_re.search(line)] + rawlines['parsed'] = self._includeExpander(rawlines['orig']) + for conftype, cfg in rawlines.items(): + _confdict = copy.deepcopy(self.defaults) + _sect = None + for line in cfg: + if self._sct_re.search(line): + _sect = self._sct_re.search(line).group('sect') + if _sect not in _confdict.keys(): + _confdict[_sect] = OrderedDict() + elif self._kv_re.search(line): + r = self._kv_re.search(line) + k = r.group('key') + v = r.group('value') + if k in self._noval_keys: + _confdict[_sect][k] = None + elif k in self._single_keys: + _confdict[_sect][k] = v + elif k in self._list_keys: + if k not in _confdict[_sect].keys(): + _confdict[_sect][k] = [] + _confdict[_sect][k].append(v) + if _confdict['options']['Architecture'] == 'auto': + _confdict['options']['Architecture'] = os.uname().machine + self.conf[conftype] = copy.deepcopy(_confdict) + return(None) + + def writeConf(self): + with open(self.confpath, 'w') as fh: + fh.write(self.j2_conf.render(cfg = self.conf)) + with open(self.mirrorlstpath, 'w') as fh: + fh.write(self.j2_mirror.render(mirrors = self.mirrors)) + return(None) diff --git a/aif/pacman/keyring.py b/aif/software/keyring.py similarity index 99% rename from aif/pacman/keyring.py rename to aif/software/keyring.py index c691905..e80b97e 100644 --- a/aif/pacman/keyring.py +++ b/aif/software/keyring.py @@ -8,6 +8,8 @@ import gpg # We don't use utils.gpg_handler because this is pretty much all procedural. +# Though, maybe add e.g. TofuDB stuff to it, and subclass it here? +# TODO. _logger = logging.getLogger(__name__) diff --git a/aif/software/mirrorlist.j2 b/aif/software/mirrorlist.j2 new file mode 100644 index 0000000..7e6929e --- /dev/null +++ b/aif/software/mirrorlist.j2 @@ -0,0 +1,5 @@ +# Generated by AIF-NG. +# See /etc/pacman.d/mirrorlist.bak for original version. +{%- for mirror in mirrors %} +Server = {{ mirror }} +{%- endfor %} diff --git a/aif/software/objtypes.py b/aif/software/objtypes.py new file mode 100644 index 0000000..8e43dda --- /dev/null +++ b/aif/software/objtypes.py @@ -0,0 +1,72 @@ +import logging +import os +import re +## +from lxml import etree + + +_logger = logging.getLogger(__name__) + + +class Mirror(object): + def __init__(self, mirror_xml, repo = None, arch = None): + self.xml = mirror_xml + _logger.debug('mirror_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) + self.uri = self.xml.text + self.real_uri = None + self.aif_uri = None + + def parse(self, chroot_base, repo, arch): + self.real_uri = self.uri.replace('$repo', repo).replace('$arch', arch) + if self.uri.startswith('file://'): + self.aif_uri = os.path.join(chroot_base, re.sub(r'^file:///?', '')) + + +class Package(object): + def __init__(self, package_xml): + self.xml = package_xml + _logger.debug('package_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) + self.name = self.xml.text + self.repo = self.xml.attrib.get('repo') + if self.repo: + self.qualified_name = '{0}/{1}'.format(self.repo, self.name) + else: + self.qualified_name = self.name + + +class Repo(object): + def __init__(self, chroot_base, repo_xml, arch = 'x86_64'): + # TODO: support Usage? ("REPOSITORY SECTIONS", pacman.conf(5)) + self.xml = repo_xml + _logger.debug('repo_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) + # TODO: SigLevels?! + self.name = self.xml.attrib['name'] + self.conflines = {} + self.mirrors = [] + self.parsed_mirrors = [] + _mirrors = self.xml.xpath('mirror|include') # "Server" and "Include" respectively in pyalpm lingo. + if _mirrors: + for m in _mirrors: + k = m.tag.title() + if k == 'Mirror': + k = 'Server' + if k not in self.conflines.keys(): + self.conflines[k] = [] + self.conflines[k].append(m.text) + # TODO; better parsing here. handle in config.py? + # if m.tag == 'include': + # # TODO: We only support one level of includes. Pacman supports unlimited nesting? of includes. + # file_uri = os.path.join(chroot_base, re.sub(r'^/?', '', m.text)) + # if not os.path.isfile(file_uri): + # _logger.error('Include file ({0}) does not exist: {1}'.format(m.text, file_uri)) + # raise FileNotFoundError('Include file does not exist') + # with open(file_uri, 'r') as fh: + # for line in fh.read().splitlines(): + else: + # Default (mirrorlist) + self.conflines['Include'] = ['file:///etc/pacman.d/mirrorlist'] + self.enabled = (True if self.xml.attrib.get('enabled', 'true') in ('1', 'true') else False) + self.siglevel = self.xml.attrib.get('sigLevel') + # self.real_uri = None + # if self.uri: + # self.real_uri = self.uri.replace('$repo', self.name).replace('$arch', arch) diff --git a/aif/software/pacman.conf.j2 b/aif/software/pacman.conf.j2 new file mode 100644 index 0000000..58e1b29 --- /dev/null +++ b/aif/software/pacman.conf.j2 @@ -0,0 +1,16 @@ +# Generated by AIF-NG. +# See /etc/pacman.conf.bak for original version. +{%- for section, kv in cfg.items() %} +[{{ section }}] + {%- for key, value in kv.items() %} + {%- if value is none %} +{{ key }} + {%- elif value|isList %} + {%- for val in value %} +{{ key }} = {{ val }} + {%- endfor %} + {%- else %} +{{ key }} = {{ val }} + {%- endif %} + {%- endfor %} +{% endfor %} diff --git a/aif/pacman/__init__.py b/aif/software/pacman.py similarity index 64% rename from aif/pacman/__init__.py rename to aif/software/pacman.py index 83d6609..466cae4 100644 --- a/aif/pacman/__init__.py +++ b/aif/software/pacman.py @@ -1,18 +1,15 @@ # We can manually bootstrap and alter pacman's keyring. But check the bootstrap tarball; we might not need to. # TODO. -import configparser import logging import os import re ## import pyalpm -import gpg from lxml import etree ## -from . import _common from . import keyring - +from . import objtypes _logger = logging.getLogger(__name__) @@ -24,32 +21,6 @@ _logger = logging.getLogger(__name__) # and have a write function to write out a mirror list to a specified location. -class Mirror(object): - def __init__(self, mirror_xml, repo = None, arch = None): - self.xml = mirror_xml - _logger.debug('mirror_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) - self.uri = self.xml.text - self.real_uri = None - self.aif_uri = None - - def parse(self, chroot_base, repo, arch): - self.real_uri = self.uri.replace('$repo', repo).replace('$arch', arch) - if self.uri.startswith('file://'): - self.aif_uri = os.path.join(chroot_base, re.sub(r'^file:///?', '')) - - -class Package(object): - def __init__(self, package_xml): - self.xml = package_xml - _logger.debug('package_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) - self.name = self.xml.text - self.repo = self.xml.attrib.get('repo') - if self.repo: - self.qualified_name = '{0}/{1}'.format(self.repo, self.name) - else: - self.qualified_name = self.name - - class PackageManager(object): def __init__(self, chroot_base, pacman_xml): self.xml = pacman_xml @@ -118,7 +89,7 @@ class PackageManager(object): with open(_mirrorlist, 'a') as fh: fh.write('\n# Added by AIF-NG.\n') for m in mirrors.findall('mirror'): - mirror = Mirror(m) + mirror = objtypes.Mirror(m) self.mirrorlist.append(mirror) fh.write('Server = {0}\n'.format(mirror.uri)) _logger.info('Appended: {0}'.format(_mirrorlist)) @@ -130,7 +101,7 @@ class PackageManager(object): with open(_conf, 'a') as fh: fh.write('\n# Added by AIF-NG.\n') for r in repos.findall('repo'): - repo = Repo(self.chroot_base, r) + repo = objtypes.Repo(self.chroot_base, r) if repo.enabled: fh.write('[{0}]\n'.format(repo.name)) if repo.siglevel: @@ -150,38 +121,3 @@ class PackageManager(object): self.repos.append(repo) _logger.info('Appended: {0}'.format(_conf)) return(None) - - -class Repo(object): - def __init__(self, chroot_base, repo_xml, arch = 'x86_64'): - # TODO: support Usage? ("REPOSITORY SECTIONS", pacman.conf(5)) - self.xml = repo_xml - _logger.debug('repo_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8'))) - # TODO: SigLevels?! - self.name = self.xml.attrib['name'] - self.mirrors = {} - self.parsed_mirrors = [] - _mirrors = self.xml.xpath('mirror|include') # "Server" and "Include" respectively in pyalpm lingo. - if _mirrors: - for m in _mirrors: - k = m.tag.title() - if k == 'Mirror': - k = 'Server' - if k not in self.mirrors.keys(): - self.mirrors[k] = [] - self.mirrors[k].append(m.text) - if m.tag == 'include': - file_uri = os.path.join(chroot_base, re.sub(r'^file:///?', '', m.text)) - if not os.path.isfile(file_uri): - _logger.error('Include file ({0}) does not exist: {1}'.format(m.text, file_uri)) - raise FileNotFoundError('Include file does not exist') - with open(file_uri, 'r') as fh: - for line in fh.read().splitlines(): - else: - # Default (mirrorlist) - self.mirrors['Include'] = ['file:///etc/pacman.d/mirrorlist'] - self.enabled = (True if self.xml.attrib.get('enabled', 'true') in ('1', 'true') else False) - self.siglevel = self.xml.attrib.get('sigLevel') - self.real_uri = None - if self.uri: - self.real_uri = self.uri.replace('$repo', self.name).replace('$arch', arch)