Arch_Repo_Builder/build.py

#!/usr/bin/env python3

# TODO: make as flexible as the <rpms>:/bin/build.py (flesh out args), logging, etc.

import argparse
import datetime
import copy
import io
import os
import pathlib
import re
import shutil
import subprocess
import tarfile
import tempfile
import warnings
##
import gpg
import requests
from lxml import etree


# TODO: track which versions are built so we don't need to consistently rebuild ALL packages
# TODO: logging
# TODO: should this be a configuration option?
aurbase = 'https://aur.archlinux.org'

_dflts = {'cfgfile': '~/.config/optools/arch/autorepo.xml'}


class Repo(object):
    def __init__(self, repo_xml):
        self.repo_xml = repo_xml
        self.gpg = None

    def _initSigner(self):

        self.gpg = gpg.Context()

        # Just grab the first private key until we flesh this out.
        for k in self.gpg.keylist(secret = True):
            if k.can_sign:
                self.gpg.signers = [k]
                break
        return()

class Packager(object):
    def __init__(self, cfgfile = _dflts['cfgfile'], validate = True, *args, **kwargs):
        self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
        self.cfg = None
        self.xml = None
        self.schema = None
        self._initCfg(validate = validate)
        self.origdir = os.path.abspath(os.path.expanduser(os.getcwd()))
        self.gpg = None
        self._initSigner()

    def _initCfg(self, validate = True):
        with open(self.cfgfile, 'rb') as f:
            self.xml = etree.parse(f)
        self.xml.xinclude()
        self.cfg = self.xml.getroot()
        if validate:
            if not self.schema:
                from urllib.request import urlopen
                xsi = self.cfg.nsmap.get('xsi', 'http://www.w3.org/2001/XMLSchema-instance')
                schemaLocation = '{{{0}}}schemaLocation'.format(xsi)
                schemaURL = self.cfg.attrib.get(schemaLocation,
                                                ('http://git.square-r00t.net/Arch_Repo_Builder/plain/archrepo.xsd'))
                with urlopen(schemaURL) as url:
                    self.schema = url.read()
            self.schema = etree.XMLSchema(etree.XML(self.schema))
            self.schema.assertValid(self.xml)
        return()

    def buildPkgs(self, auronly = None):
        for p in self.args['pkgs']:
            print(p)
            extract_dir = tempfile.mkdtemp(prefix = '.pkgbuilder.{0}-'.format(p))
            sub_extract_dir = os.path.join(extract_dir, p)
            has_pkg = False
            if not auronly:
                has_pkg = self._getLocal(p, extract_dir)
            if not has_pkg:
                has_pkg = self._getAUR(p, extract_dir)
            if not has_pkg:
                warnings.warn('Could not find package {0}; skipping...'.format(p))
                continue
            # We get a list of files to compare.
            prebuild_files = []
            postbuild_files = []
            for root, dirs, files in os.walk(sub_extract_dir):
                for f in files:
                    prebuild_files.append(os.path.join(root, f))
            os.chdir(os.path.join(extract_dir, p))
            # customizepkg-scripting in AUR
            try:
                custpkg_out = subprocess.run(['/usr/bin/customizepkg',
                                              '-m'],
                                             stdout = subprocess.PIPE,
                                             stderr = subprocess.PIPE)
            except FileNotFoundError:
                pass  # Not installed
            build_out = subprocess.run(['/usr/bin/multilib-build',
                                        '-c',
                                        '--',
                                        '--',
                                        '--skippgpcheck',
                                        '--syncdeps',
                                        '--noconfirm',
                                        '--log',
                                        '--holdver',
                                        '--skipinteg'],
                                       stdout = subprocess.PIPE,
                                       stderr = subprocess.PIPE)
            # with open('/tmp/build.log-{0}'.format(p), 'w') as f:
            #     f.write(build_out.stdout.decode('utf-8'))
            for root, dirs, files in os.walk(sub_extract_dir):
                for f in files:
                    fpath = os.path.join(root, f)
                    if fpath in prebuild_files:
                        continue
                    if fpath.endswith('.log'):
                        continue
                    postbuild_files.append(fpath)
            postbuild_files = [i for i in postbuild_files if i.endswith('.pkg.tar.xz')]
            if len(postbuild_files) != 1:
                warnings.warn('Could not reliably find a built package for {0}; skipping'.format(p))
            else:
                fdest = os.path.join(self.args['destdir'],
                                     os.path.basename(postbuild_files[0]))
                if os.path.isfile(fdest):
                    os.remove(fdest)
                shutil.move(postbuild_files[0], fdest)
                self._sign(fdest)
            os.chdir(self.origdir)
            shutil.rmtree(extract_dir)
        return()

    def _getAUR(self, pkgnm, extract_dir):
        dl_url = None
        pkg_srch = requests.get(os.path.join(self.args['aurbase'],
                                             'rpc'),
                                params = {
                                    'v': 5,
                                    'type': 'search',
                                    'by': 'name',
                                    'arg': pkgnm}).json()
        for pkg in pkg_srch['results']:
            dl_url = None
            if pkg['Name'] == pkgnm:
                dl_url = os.path.join(self.args['aurbase'], re.sub('^/+', '', pkg['URLPath']))
                # dl_file = os.path.basename(pkg['URLPath'])
                break
        if not dl_url:
            warnings.warn('Could not find a download path for {0}; skipping'.format(pkgnm))
            return(False)
        with requests.get(dl_url, stream = True) as url:
            with tarfile.open(mode = 'r|*', fileobj = io.BytesIO(url.content)) as tar:
                tar.extractall(extract_dir)
        return(True)

    def _getLocal(self, pkgnm, extract_dir):
        curfile = os.path.realpath(os.path.abspath(os.path.expanduser(__file__)))
        localpkg_dir = os.path.abspath(os.path.join(os.path.dirname(curfile),
                                                    '..',
                                                    'local_pkgs'))
        pkgbuild_dir = os.path.join(localpkg_dir,
                                    pkgnm)
        if not os.path.isdir(pkgbuild_dir):
            return(False)
        shutil.copytree(pkgbuild_dir, os.path.join(extract_dir, pkgnm))
        return(True)

    def _sign(self, pkgfile, passphrase = None):
        sigfile = '{0}.sig'.format(pkgfile)
        with open(pkgfile, 'rb') as pkg:
            with open(sigfile, 'wb') as sig:
                # We want ascii-armoured detached sigs
                sig.write(self.gpg.sign(pkg.read(), mode = gpg.constants.SIG_MODE_DETACH)[0])
        return()

    def createRepo(self):
        pkgfiles = []
        for root, dirs, files in os.walk(self.args['destdir']):
            for f in files:
                if f.endswith('.pkg.tar.xz'):
                    pkgfiles.append(os.path.join(root, f))
        repo_out = subprocess.run(['/usr/bin/repo-add',
                                   '-s',
                                   '-R',
                                   os.path.join(self.args['destdir'], '{0}.db.tar.xz'.format(self.args['reponame'])),
                                   *pkgfiles],
                                  stdout = subprocess.PIPE,
                                  stderr = subprocess.PIPE)
        return()


def parseArgs():
    args = argparse.ArgumentParser(description = 'Build Pacman packages and update a local repository')
    args.add_argument('-n', '--no-validate',
                      dest = 'validate',
                      action = 'store_false',
                      help = ('If specified, do NOT attempt to validate the config file (-c/--config)'))
    args.add_argument('-c', '--config',
                      dest = 'cfgfile',
                      default = _dflts['cfgfile'],
                      help = ('The path to the configuration file. Default: {0}').format(_dflts['cfgfile']))
    return(args)

def main():
    args = parseArgs().parse_args()
    varargs = vars(args)
    pkgr = Packager(**varargs)
    pkgr.buildPkgs(auronly = varargs['auronly'])
    pkgr.createRepo()
    return()

if __name__ == '__main__':
    main()