updating some pacman stuff. need to finish objtypes.Repo and might need to tweak config writer.

aif/__init__.py

@@ -14,7 +14,7 @@ from . import system
 from . import config
 from . import envsetup
 from . import network
-from . import pacman
+from . import software
 
 
 _logger = logging.getLogger('AIF')

aif/pacman/_common.py

@@ -1,22 +0,0 @@
-import configparser
-import logging
-from collections import OrderedDict
-
-
-# TODO: Add pacman.conf parsing?
-
-
-_logger = logging.getLogger('pacman:_common')
-
-
-class MultiOrderedDict(OrderedDict):
-    # Thanks, dude: https://stackoverflow.com/a/38286559/733214
-    def __setitem__(self, key, value):
-        if key in self:
-            if isinstance(value, list):
-                self[key].extend(value)
-                return(None)
-            elif isinstance(value, str):
-                if len(self[key]) > 1:
-                    return(None)
-        super(MultiOrderedDict, self).__setitem__(key, value)

aif/software/__init__.py

@@ -0,0 +1,4 @@
+from . import config
+from . import keyring
+from . import objtypes
+from . import pacman

aif/software/config.py

@@ -0,0 +1,124 @@
+import copy
+import logging
+import os
+import re
+import shutil
+from collections import OrderedDict
+##
+import jinja2
+##
+import aif.utils
+
+
+_logger = logging.getLogger(__name__)
+
+
+class PacmanConfig(object):
+    _sct_re = re.compile(r'^\s*\[(?P<sect>[^]]+)\]\s*$')
+    _kv_re = re.compile(r'^\s*(?P<key>[^\s=[]+)((?:\s*=\s*)(?P<value>.*))?$')
+    _skipline_re = re.compile(r'^\s*(#.*)?$')
+    # TODO: Append mirrors/repos to pacman.conf here before we parse?
+    # I copy a log of logic from pycman/config.py here.
+    _list_keys = ('CacheDir', 'HookDir', 'HoldPkg', 'SyncFirst', 'IgnoreGroup', 'IgnorePkg', 'NoExtract', 'NoUpgrade',
+                  'Server')
+    _single_keys = ('RootDir', 'DBPath', 'GPGDir', 'LogFile', 'Architecture', 'XferCommand', 'CleanMethod', 'SigLevel',
+                    'LocalFileSigLevel', 'RemoteFileSigLevel')
+    _noval_keys = ('UseSyslog', 'ShowSize', 'TotalDownload', 'CheckSpace', 'VerbosePkgLists', 'ILoveCandy', 'Color',
+                   'DisableDownloadTimeout')
+    # These are the default (commented-out) values in the stock /etc/pacman.conf as of January 5, 2020.
+    defaults = OrderedDict({'options': {'Architecture': 'auto',
+                                        'CacheDir': '/var/cache/pacman/pkg/',
+                                        'CheckSpace': None,
+                                        'CleanMethod': 'KeepInstalled',
+                                        # 'Color': None,
+                                        'DBPath': '/var/lib/pacman/',
+                                        'GPGDir': '/etc/pacman.d/gnupg/',
+                                        'HoldPkg': 'pacman glibc',
+                                        'HookDir': '/etc/pacman.d/hooks/',
+                                        'IgnoreGroup': [],
+                                        'IgnorePkg': [],
+                                        'LocalFileSigLevel': ['Optional'],
+                                        'LogFile': '/var/log/pacman.log',
+                                        'NoExtract': [],
+                                        'NoUpgrade': [],
+                                        'RemoteFileSigLevel': ['Required'],
+                                        'RootDir': '/',
+                                        'SigLevel': ['Required', 'DatabaseOptional'],
+                                        # 'TotalDownload': None,
+                                        # 'UseSyslog': None,
+                                        # 'VerbosePkgLists': None,
+                                        'XferCommand': '/usr/bin/curl -L -C - -f -o %o %u'},
+                            # These should be explicitly included in the AIF config.
+                            # 'core': {'Include': '/etc/pacman.d/mirrorlist'},
+                            # 'extra': {'Include': '/etc/pacman.d/mirrorlist'},
+                            # 'community': {'Include': '/etc/pacman.d/mirrorlist'}
+                            })
+
+    def __init__(self, chroot_base, confpath = '/etc/pacman.conf'):
+        self.chroot_base = chroot_base
+        self.confpath = os.path.join(self.chroot_base, re.sub(r'^/+', '', confpath))
+        self.confbak = '{0}.bak'.format(self.confpath)
+        self.mirrorlstpath = os.path.join(self.chroot_base, 'etc', 'pacman.d', 'mirrorlist')
+        self.mirrorlstbak = '{0}.bak'.format(self.mirrorlstpath)
+        if not os.path.isfile(self.confbak):
+            shutil.copy2(self.confpath, self.confbak)
+            _logger.info('Copied: {0} => {1}'.format(self.confpath, self.confbak))
+        if not os.path.isfile(self.mirrorlstbak):
+            shutil.copy2(self.mirrorlstpath, self.mirrorlstbak)
+            _logger.info('Copied: {0} => {1}'.format(self.mirrorlstpath, self.mirrorlstbak))
+        self.j2_env = jinja2.Environment(loader = jinja2.FileSystemLoader(searchpath = './'))
+        self.j2_env.filters.update(aif.utils.j2_filters)
+        self.j2_conf = self.j2_env.get_template('pacman.conf.j2')
+        self.j2_mirror = self.j2_env.get_template('mirrorlist.j2')
+        self.conf = None
+        self.mirrors = []
+
+    def _includeExpander(self, lines):
+        curlines = []
+        for line in lines:
+            r = self._kv_re.search(line)
+            if r and (r.group('key') == 'Include') and r.group('value'):
+                path = os.path.join(self.chroot_base, re.sub(r'^/?', '', r.group('path')))
+                with open(path, 'r') as fh:
+                    curlines.extend(self._includeExpander(fh.read().splitlines()))
+            else:
+                curlines.append(line)
+        return(curlines)
+
+    def parse(self, defaults = True):
+        self.conf = OrderedDict()
+        rawlines = {}
+        with open(self.confpath, 'r') as fh:
+            rawlines['orig'] = [line for line in fh.read().splitlines() if not self._skipline_re.search(line)]
+        rawlines['parsed'] = self._includeExpander(rawlines['orig'])
+        for conftype, cfg in rawlines.items():
+            _confdict = copy.deepcopy(self.defaults)
+            _sect = None
+            for line in cfg:
+                if self._sct_re.search(line):
+                    _sect = self._sct_re.search(line).group('sect')
+                    if _sect not in _confdict.keys():
+                        _confdict[_sect] = OrderedDict()
+                elif self._kv_re.search(line):
+                    r = self._kv_re.search(line)
+                    k = r.group('key')
+                    v = r.group('value')
+                    if k in self._noval_keys:
+                        _confdict[_sect][k] = None
+                    elif k in self._single_keys:
+                        _confdict[_sect][k] = v
+                    elif k in self._list_keys:
+                        if k not in _confdict[_sect].keys():
+                            _confdict[_sect][k] = []
+                        _confdict[_sect][k].append(v)
+            if _confdict['options']['Architecture'] == 'auto':
+                _confdict['options']['Architecture'] = os.uname().machine
+            self.conf[conftype] = copy.deepcopy(_confdict)
+        return(None)
+
+    def writeConf(self):
+        with open(self.confpath, 'w') as fh:
+            fh.write(self.j2_conf.render(cfg = self.conf))
+        with open(self.mirrorlstpath, 'w') as fh:
+            fh.write(self.j2_mirror.render(mirrors = self.mirrors))
+        return(None)

aif/software/keyring.py

@@ -8,6 +8,8 @@ import gpg
 
 
 # We don't use utils.gpg_handler because this is pretty much all procedural.
+# Though, maybe add e.g. TofuDB stuff to it, and subclass it here?
+# TODO.
 
 _logger = logging.getLogger(__name__)
 

aif/software/mirrorlist.j2

@@ -0,0 +1,5 @@
+# Generated by AIF-NG.
+# See /etc/pacman.d/mirrorlist.bak for original version.
+{%- for mirror in mirrors %}
+Server = {{ mirror }}
+{%- endfor %}

aif/software/objtypes.py

@@ -0,0 +1,72 @@
+import logging
+import os
+import re
+##
+from lxml import etree
+
+
+_logger = logging.getLogger(__name__)
+
+
+class Mirror(object):
+    def __init__(self, mirror_xml, repo = None, arch = None):
+        self.xml = mirror_xml
+        _logger.debug('mirror_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
+        self.uri = self.xml.text
+        self.real_uri = None
+        self.aif_uri = None
+
+    def parse(self, chroot_base, repo, arch):
+        self.real_uri = self.uri.replace('$repo', repo).replace('$arch', arch)
+        if self.uri.startswith('file://'):
+            self.aif_uri = os.path.join(chroot_base, re.sub(r'^file:///?', ''))
+
+
+class Package(object):
+    def __init__(self, package_xml):
+        self.xml = package_xml
+        _logger.debug('package_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
+        self.name = self.xml.text
+        self.repo = self.xml.attrib.get('repo')
+        if self.repo:
+            self.qualified_name = '{0}/{1}'.format(self.repo, self.name)
+        else:
+            self.qualified_name = self.name
+
+
+class Repo(object):
+    def __init__(self, chroot_base, repo_xml, arch = 'x86_64'):
+        # TODO: support Usage? ("REPOSITORY SECTIONS", pacman.conf(5))
+        self.xml = repo_xml
+        _logger.debug('repo_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
+        # TODO: SigLevels?!
+        self.name = self.xml.attrib['name']
+        self.conflines = {}
+        self.mirrors = []
+        self.parsed_mirrors = []
+        _mirrors = self.xml.xpath('mirror|include')  # "Server" and "Include" respectively in pyalpm lingo.
+        if _mirrors:
+            for m in _mirrors:
+                k = m.tag.title()
+                if k == 'Mirror':
+                    k = 'Server'
+                if k not in self.conflines.keys():
+                    self.conflines[k] = []
+                self.conflines[k].append(m.text)
+                # TODO; better parsing here. handle in config.py?
+                # if m.tag == 'include':
+                #     # TODO: We only support one level of includes. Pacman supports unlimited nesting? of includes.
+                #     file_uri = os.path.join(chroot_base, re.sub(r'^/?', '', m.text))
+                #     if not os.path.isfile(file_uri):
+                #         _logger.error('Include file ({0}) does not exist: {1}'.format(m.text, file_uri))
+                #         raise FileNotFoundError('Include file does not exist')
+                #     with open(file_uri, 'r') as fh:
+                #         for line in fh.read().splitlines():
+        else:
+            # Default (mirrorlist)
+            self.conflines['Include'] = ['file:///etc/pacman.d/mirrorlist']
+        self.enabled = (True if self.xml.attrib.get('enabled', 'true') in ('1', 'true') else False)
+        self.siglevel = self.xml.attrib.get('sigLevel')
+        # self.real_uri = None
+        # if self.uri:
+        #     self.real_uri = self.uri.replace('$repo', self.name).replace('$arch', arch)

aif/software/pacman.conf.j2

@@ -0,0 +1,16 @@
+# Generated by AIF-NG.
+# See /etc/pacman.conf.bak for original version.
+{%- for section, kv in cfg.items() %}
+[{{ section }}]
+  {%- for key, value in kv.items() %}
+    {%- if value is none %}
+{{ key }}
+    {%- elif value|isList %}
+      {%- for val in value %}
+{{ key }} = {{ val }}
+      {%- endfor %}
+    {%- else %}
+{{ key }} = {{ val }}
+	{%- endif %}
+  {%- endfor %}
+{% endfor %}

aif/software/pacman.py

@@ -1,18 +1,15 @@
 # We can manually bootstrap and alter pacman's keyring. But check the bootstrap tarball; we might not need to.
 # TODO.
 
-import configparser
 import logging
 import os
 import re
 ##
 import pyalpm
-import gpg
 from lxml import etree
 ##
-from . import _common
 from . import keyring
-
+from . import objtypes
 
 _logger = logging.getLogger(__name__)
 
@@ -24,32 +21,6 @@ _logger = logging.getLogger(__name__)
 #         and have a write function to write out a mirror list to a specified location.
 
 
-class Mirror(object):
-    def __init__(self, mirror_xml, repo = None, arch = None):
-        self.xml = mirror_xml
-        _logger.debug('mirror_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
-        self.uri = self.xml.text
-        self.real_uri = None
-        self.aif_uri = None
-
-    def parse(self, chroot_base, repo, arch):
-        self.real_uri = self.uri.replace('$repo', repo).replace('$arch', arch)
-        if self.uri.startswith('file://'):
-            self.aif_uri = os.path.join(chroot_base, re.sub(r'^file:///?', ''))
-
-
-class Package(object):
-    def __init__(self, package_xml):
-        self.xml = package_xml
-        _logger.debug('package_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
-        self.name = self.xml.text
-        self.repo = self.xml.attrib.get('repo')
-        if self.repo:
-            self.qualified_name = '{0}/{1}'.format(self.repo, self.name)
-        else:
-            self.qualified_name = self.name
-
-
 class PackageManager(object):
     def __init__(self, chroot_base, pacman_xml):
         self.xml = pacman_xml
@@ -118,7 +89,7 @@ class PackageManager(object):
             with open(_mirrorlist, 'a') as fh:
                 fh.write('\n# Added by AIF-NG.\n')
                 for m in mirrors.findall('mirror'):
-                    mirror = Mirror(m)
+                    mirror = objtypes.Mirror(m)
                     self.mirrorlist.append(mirror)
                     fh.write('Server = {0}\n'.format(mirror.uri))
             _logger.info('Appended: {0}'.format(_mirrorlist))
@@ -130,7 +101,7 @@ class PackageManager(object):
         with open(_conf, 'a') as fh:
             fh.write('\n# Added by AIF-NG.\n')
             for r in repos.findall('repo'):
-                repo = Repo(self.chroot_base, r)
+                repo = objtypes.Repo(self.chroot_base, r)
                 if repo.enabled:
                     fh.write('[{0}]\n'.format(repo.name))
                     if repo.siglevel:
@@ -150,38 +121,3 @@ class PackageManager(object):
                 self.repos.append(repo)
         _logger.info('Appended: {0}'.format(_conf))
         return(None)
-
-
-class Repo(object):
-    def __init__(self, chroot_base, repo_xml, arch = 'x86_64'):
-        # TODO: support Usage? ("REPOSITORY SECTIONS", pacman.conf(5))
-        self.xml = repo_xml
-        _logger.debug('repo_xml: {0}'.format(etree.tostring(self.xml, with_tail = False).decode('utf-8')))
-        # TODO: SigLevels?!
-        self.name = self.xml.attrib['name']
-        self.mirrors = {}
-        self.parsed_mirrors = []
-        _mirrors = self.xml.xpath('mirror|include')  # "Server" and "Include" respectively in pyalpm lingo.
-        if _mirrors:
-            for m in _mirrors:
-                k = m.tag.title()
-                if k == 'Mirror':
-                    k = 'Server'
-                if k not in self.mirrors.keys():
-                    self.mirrors[k] = []
-                self.mirrors[k].append(m.text)
-                if m.tag == 'include':
-                    file_uri = os.path.join(chroot_base, re.sub(r'^file:///?', '', m.text))
-                    if not os.path.isfile(file_uri):
-                        _logger.error('Include file ({0}) does not exist: {1}'.format(m.text, file_uri))
-                        raise FileNotFoundError('Include file does not exist')
-                    with open(file_uri, 'r') as fh:
-                        for line in fh.read().splitlines():
-        else:
-            # Default (mirrorlist)
-            self.mirrors['Include'] = ['file:///etc/pacman.d/mirrorlist']
-        self.enabled = (True if self.xml.attrib.get('enabled', 'true') in ('1', 'true') else False)
-        self.siglevel = self.xml.attrib.get('sigLevel')
-        self.real_uri = None
-        if self.uri:
-            self.real_uri = self.uri.replace('$repo', self.name).replace('$arch', arch)