From 72298d7a4c7e0348ee3071b2413564526405437b Mon Sep 17 00:00:00 2001
From: brent s <r00t@square-r00t.net>
Date: Thu, 19 Sep 2019 02:01:57 -0400
Subject: [PATCH] moving autorepo to Arch_Repo_Builder repo

---
 arch/autorepo/autorepo.xsd     | 141 ------------------------
 arch/autorepo/build.py         | 191 ---------------------------------
 arch/autorepo/example.pkgs.xml | 112 -------------------
 arch/autorepo/readme.txt       |   9 --
 arch/autorepo/regen_sudoers.py |  33 ------
 arch/autorepo/sync.sh          |  15 ---
 6 files changed, 501 deletions(-)
 delete mode 100644 arch/autorepo/autorepo.xsd
 delete mode 100755 arch/autorepo/build.py
 delete mode 100644 arch/autorepo/example.pkgs.xml
 delete mode 100644 arch/autorepo/readme.txt
 delete mode 100755 arch/autorepo/regen_sudoers.py
 delete mode 100755 arch/autorepo/sync.sh

diff --git a/arch/autorepo/autorepo.xsd b/arch/autorepo/autorepo.xsd
deleted file mode 100644
index 27622ac..0000000
--- a/arch/autorepo/autorepo.xsd
+++ /dev/null
@@ -1,141 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
-           targetNamespace="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
-           xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/tree/"
-           xmlns:archrepo="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
-           elementFormDefault="qualified"
-           attributeFormDefault="unqualified">
-
-    <xs:simpleType name="t_posixUserGroup">
-        <xs:restriction base="xs:token">
-            <xs:pattern value="[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}$)"/>
-            <xs:pattern value="%same"/>
-            <xs:whiteSpace value="collapse"/>
-        </xs:restriction>
-    </xs:simpleType>
-
-    <xs:simpleType name="t_posixMode">
-        <xs:restriction base="xs:positiveInteger">
-            <xs:pattern value="[0-7]?[0-7]{3}"/>
-            <xs:whiteSpace value="collapse"/>
-        </xs:restriction>
-    </xs:simpleType>
-
-    <xs:simpleType name="t_path">
-        <xs:restriction base="xs:string">
-            <xs:pattern value="(/|~/)?([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
-            <xs:whiteSpace value="collapse"/>
-        </xs:restriction>
-    </xs:simpleType>
-
-    <xs:simpleType name="t_port">
-        <xs:restriction base="xs:positiveInteger">
-            <!-- MAN I wish XSD let you validate based on numerical range. -->
-            <!-- https://stackoverflow.com/a/40213676/733214 -->
-            <xs:pattern value="([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])"/>
-        </xs:restriction>
-    </xs:simpleType>
-
-    <xs:complexType name="t_localMirror">
-        <xs:simpleContent>
-            <xs:extension base="xs:anyURI">
-                <xs:attribute name="user" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
-                <xs:attribute name="group" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
-                <xs:attribute name="fileMode" type="archrepo:t_posixMode" use="optional" default="0600"/>
-                <xs:attribute name="dirMode" type="archrepo:t_posixMode" use="optional" default="0700"/>
-            </xs:extension>
-        </xs:simpleContent>
-    </xs:complexType>
-
-    <xs:complexType name="t_remoteMirror">
-        <xs:simpleContent>
-            <xs:extension base="xs:anyURI">
-                <xs:attribute name="user" type="archrepo:t_posixUserGroup"
-                              default="%same" use="optional"/>
-                <xs:attribute name="server" type="xs:NMTOKEN" use="required"/>
-                <xs:attribute name="fileMode" type="archrepo:t_posixMode"
-                              use="optional" default="0600"/>
-                <xs:attribute name="dirMode" type="archrepo:t_posixMode"
-                              use="optional" default="0700"/>
-                <xs:attribute name="port" type="archrepo:t_port"
-                              default="22" use="optional"/>
-                <xs:attribute name="key" type="archrepo:t_path"
-                              default="~/.ssh/id_rsa" use="optional"/>
-                <xs:attribute name="remoteUser" type="archrepo:t_posixUserGroup"
-                              default="%same" use="optional"/>
-                <xs:attribute name="remoteGroup" type="archrepo:t_posixUserGroup"
-                              default="%same" use="optional"/>
-            </xs:extension>
-        </xs:simpleContent>
-    </xs:complexType>
-
-    <xs:simpleType name="t_gpgKeyID">
-        <xs:restriction base="xs:string">
-            <xs:pattern value="(0[Xx])?[0-9A-Fa-f]{40}"/>
-            <xs:pattern value="(0[Xx])?[0-9A-Fa-f]{8}"/>
-            <xs:pattern value="(0[Xx])?([0-9A-Fa-f]{4} ?){5} *([0-9A-Fa-f]{4}){5}"/>
-            <xs:whiteSpace value="collapse"/>
-        </xs:restriction>
-    </xs:simpleType>
-
-    <xs:element name="archrepo">
-        <xs:complexType>
-            <xs:choice>
-                <xs:element name="repo" minOccurs="1" maxOccurs="unbounded">
-                    <xs:complexType>
-                        <xs:all minOccurs="1">
-                            <xs:element name="mirrors" minOccurs="1" maxOccurs="1">
-                                <xs:complexType>
-                                    <xs:choice minOccurs="1" maxOccurs="unbounded">
-                                        <xs:element name="localMirror"
-                                                    maxOccurs="unbounded"
-                                                    type="archrepo:t_localMirror"/>
-                                        <xs:element name="remoteMirror"
-                                                    maxOccurs="unbounded"
-                                                    type="archrepo:t_remoteMirror"/>
-                                    </xs:choice>
-                                </xs:complexType>
-                            </xs:element>
-                            <xs:element name="packages" minOccurs="1">
-                                <xs:complexType>
-                                    <xs:choice minOccurs="1" maxOccurs="unbounded">
-                                        <xs:element name="aur"
-                                                    maxOccurs="unbounded">
-                                            <xs:complexType>
-                                                <xs:simpleContent>
-                                                    <xs:extension base="xs:string">
-                                                        <xs:attribute name="alwaysBuild" default="true"
-                                                                      type="xs:boolean" use="optional"/>
-                                                    </xs:extension>
-                                                </xs:simpleContent>
-                                            </xs:complexType>
-                                        </xs:element>
-                                        <xs:element name="pkgbuild"
-                                                    maxOccurs="unbounded">
-                                            <xs:complexType>
-                                                <xs:simpleContent>
-                                                    <xs:extension base="xs:token">
-                                                        <xs:attribute name="path" type="archrepo:t_path"
-                                                                      default="." use="optional"/>
-                                                        <xs:attribute name="alwaysBuild" default="true"
-                                                                      type="xs:boolean" use="optional"/>
-                                                    </xs:extension>
-                                                </xs:simpleContent>
-                                            </xs:complexType>
-                                        </xs:element>
-                                    </xs:choice>
-                                </xs:complexType>
-                            </xs:element>
-                        </xs:all>
-                        <xs:attribute name="name" type="xs:token" use="required"/>
-                        <xs:attribute name="staging" type="archrepo:t_path" use="optional" default="."/>
-                        <xs:attribute name="signPkgs" type="xs:boolean" use="optional" default="true"/>
-                        <xs:attribute name="signDB" type="xs:boolean" use="optional" default="true"/>
-                        <xs:attribute name="gnupgHome" type="archrepo:t_path" use="optional" default="~/.gnupg"/>
-                        <xs:attribute name="gpgKeyID" type="archrepo:t_gpgKeyID" use="optional"/>
-                    </xs:complexType>
-                </xs:element>
-            </xs:choice>
-        </xs:complexType>
-    </xs:element>
-</xs:schema>
\ No newline at end of file
diff --git a/arch/autorepo/build.py b/arch/autorepo/build.py
deleted file mode 100755
index 5256069..0000000
--- a/arch/autorepo/build.py
+++ /dev/null
@@ -1,191 +0,0 @@
-#!/usr/bin/env python3
-
-# TODO: make as flexible as the <rpms>:/bin/build.py (flesh out args), logging, etc.
-
-import argparse
-import datetime
-import copy
-import io
-import os
-import pathlib
-import re
-import shutil
-import subprocess
-import tarfile
-import tempfile
-import warnings
-##
-import gpg
-import requests
-from lxml import etree
-
-
-# TODO: track which versions are built so we don't need to consistently rebuild ALL packages
-# TODO: should this be a configuration option?
-aurbase = 'https://aur.archlinux.org'
-
-_dflts = {'cfgfile': '~/.config/optools/arch/autorepo.xml'}
-
-
-class Packager(object):
-    def __init__(self, cfgfile = _dflts['cfgfile'], *args, **kwargs):
-        user_params = kwargs
-        self.args = copy.deepcopy(_dflts)
-        self.args.update(user_params)
-        self.origdir = os.path.abspath(os.path.expanduser(os.getcwd()))
-        self.gpg = None
-        self.args['destdir'] = os.path.abspath(os.path.expanduser(self.args['destdir']))
-        if not self.args['pkgs']:
-            self.args['pkgs'] = _dflts['pkgs']
-        self._initSigner()
-
-    def buildPkgs(self, auronly = None):
-        for p in self.args['pkgs']:
-            print(p)
-            extract_dir = tempfile.mkdtemp(prefix = '.pkgbuilder.{0}-'.format(p))
-            sub_extract_dir = os.path.join(extract_dir, p)
-            has_pkg = False
-            if not auronly:
-                has_pkg = self._getLocal(p, extract_dir)
-            if not has_pkg:
-                has_pkg = self._getAUR(p, extract_dir)
-            if not has_pkg:
-                warnings.warn('Could not find package {0}; skipping...'.format(p))
-                continue
-            # We get a list of files to compare.
-            prebuild_files = []
-            postbuild_files = []
-            for root, dirs, files in os.walk(sub_extract_dir):
-                for f in files:
-                    prebuild_files.append(os.path.join(root, f))
-            os.chdir(os.path.join(extract_dir, p))
-            # customizepkg-scripting in AUR
-            try:
-                custpkg_out = subprocess.run(['/usr/bin/customizepkg',
-                                              '-m'],
-                                             stdout = subprocess.PIPE,
-                                             stderr = subprocess.PIPE)
-            except FileNotFoundError:
-                pass  # Not installed
-            build_out = subprocess.run(['/usr/bin/multilib-build',
-                                        '-c',
-                                        '--',
-                                        '--',
-                                        '--skippgpcheck',
-                                        '--syncdeps',
-                                        '--noconfirm',
-                                        '--log',
-                                        '--holdver',
-                                        '--skipinteg'],
-                                       stdout = subprocess.PIPE,
-                                       stderr = subprocess.PIPE)
-            # with open('/tmp/build.log-{0}'.format(p), 'w') as f:
-            #     f.write(build_out.stdout.decode('utf-8'))
-            for root, dirs, files in os.walk(sub_extract_dir):
-                for f in files:
-                    fpath = os.path.join(root, f)
-                    if fpath in prebuild_files:
-                        continue
-                    if fpath.endswith('.log'):
-                        continue
-                    postbuild_files.append(fpath)
-            postbuild_files = [i for i in postbuild_files if i.endswith('.pkg.tar.xz')]
-            if len(postbuild_files) != 1:
-                warnings.warn('Could not reliably find a built package for {0}; skipping'.format(p))
-            else:
-                fdest = os.path.join(self.args['destdir'],
-                                     os.path.basename(postbuild_files[0]))
-                if os.path.isfile(fdest):
-                    os.remove(fdest)
-                shutil.move(postbuild_files[0], fdest)
-                self._sign(fdest)
-            os.chdir(self.origdir)
-            shutil.rmtree(extract_dir)
-        return()
-
-    def _initSigner(self):
-        self.gpg = gpg.Context()
-        # Just grab the first private key until we flesh this out.
-        for k in self.gpg.keylist(secret = True):
-            if k.can_sign:
-                self.gpg.signers = [k]
-                break
-        return()
-
-    def _getAUR(self, pkgnm, extract_dir):
-        dl_url = None
-        pkg_srch = requests.get(os.path.join(self.args['aurbase'],
-                                             'rpc'),
-                                params = {
-                                    'v': 5,
-                                    'type': 'search',
-                                    'by': 'name',
-                                    'arg': pkgnm}).json()
-        for pkg in pkg_srch['results']:
-            dl_url = None
-            if pkg['Name'] == pkgnm:
-                dl_url = os.path.join(self.args['aurbase'], re.sub('^/+', '', pkg['URLPath']))
-                # dl_file = os.path.basename(pkg['URLPath'])
-                break
-        if not dl_url:
-            warnings.warn('Could not find a download path for {0}; skipping'.format(pkgnm))
-            return(False)
-        with requests.get(dl_url, stream = True) as url:
-            with tarfile.open(mode = 'r|*', fileobj = io.BytesIO(url.content)) as tar:
-                tar.extractall(extract_dir)
-        return(True)
-
-    def _getLocal(self, pkgnm, extract_dir):
-        curfile = os.path.realpath(os.path.abspath(os.path.expanduser(__file__)))
-        localpkg_dir = os.path.abspath(os.path.join(os.path.dirname(curfile),
-                                                    '..',
-                                                    'local_pkgs'))
-        pkgbuild_dir = os.path.join(localpkg_dir,
-                                    pkgnm)
-        if not os.path.isdir(pkgbuild_dir):
-            return(False)
-        shutil.copytree(pkgbuild_dir, os.path.join(extract_dir, pkgnm))
-        return(True)
-
-    def _sign(self, pkgfile, passphrase = None):
-        sigfile = '{0}.sig'.format(pkgfile)
-        with open(pkgfile, 'rb') as pkg:
-            with open(sigfile, 'wb') as sig:
-                # We want ascii-armoured detached sigs
-                sig.write(self.gpg.sign(pkg.read(), mode = gpg.constants.SIG_MODE_DETACH)[0])
-        return()
-
-    def createRepo(self):
-        pkgfiles = []
-        for root, dirs, files in os.walk(self.args['destdir']):
-            for f in files:
-                if f.endswith('.pkg.tar.xz'):
-                    pkgfiles.append(os.path.join(root, f))
-        repo_out = subprocess.run(['/usr/bin/repo-add',
-                                   '-s',
-                                   '-R',
-                                   os.path.join(self.args['destdir'], '{0}.db.tar.xz'.format(self.args['reponame'])),
-                                   *pkgfiles],
-                                  stdout = subprocess.PIPE,
-                                  stderr = subprocess.PIPE)
-        return()
-
-
-def parseArgs():
-    args = argparse.ArgumentParser(description = 'Build Pacman packages and update a local repository')
-    args.add_argument('-c', '--config',
-                      dest = 'cfgfile',
-                      default = _dflts['cfgfile'],
-                      help = ('The path to the configuration file. Default: {0}').format(_dflts['cfgfile']))
-    return(args)
-
-def main():
-    args = parseArgs().parse_args()
-    varargs = vars(args)
-    pkgr = Packager(**varargs)
-    pkgr.buildPkgs(auronly = varargs['auronly'])
-    pkgr.createRepo()
-    return()
-
-if __name__ == '__main__':
-    main()
diff --git a/arch/autorepo/example.pkgs.xml b/arch/autorepo/example.pkgs.xml
deleted file mode 100644
index 8942485..0000000
--- a/arch/autorepo/example.pkgs.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-    The parsing supports XInclude (https://www.w3.org/TR/xinclude/).
-    You can use external XML snippets if that's easier/cleaner (it usually is).
--->
-<archrepo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-          xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
-          xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/arch/autorepo/autorepo.xsd">
-    <!--
-    The repo element contains information for each repository we should build for.
-        Attributes:
-            name: The name of the repository. This is used for the db name and to generate pacman.conf snippets.
-            staging: The path to the staging directory. This is where we will build packages and sync to mirrors from.
-            signPkgs: Either "1"/"true" or "0"/"false". Whether or not we should sign packages. See signDB, gnupgHome,
-                      and gpgKeyID.
-            signDB: Either "1"/"true" or "0"/"false". Whether or not we should sign the database files. See signPkgs,
-                    gnupgHome, and gpgKeyID.
-            gnupgHome: The path to use for the GnuPG home (GNUPGHOME environment variable).
-                       The order of preference follows:
-                            1.) gnupgHome attribute (if set)
-                            2.) $GNUPGHOME env var (if set)
-                            3.) ~/.gnupg
-                       See signPkgs, signDB, and gpgKeyID.
-            gpgKeyID: The key ID to use. It *must* have the signing ("S") capability. If it is a subkey fingerprint,
-                      that subkey will be used. If a subkey fpr is specified but lacks the signing capability, the
-                      (parent) key will be used (if it has signing capability). If no key ID/fingerprint/etc. is
-                      specified, we will use the first key with signing capability found (this should be fine if you
-                      only have one key with signing capabilities in your gnupgHome). If no suitable key is found but
-                      signing is enabled, an error will be thrown. See signPkgs, signDB, and gnupgHome.
-    -->
-    <repo
-            name="testrepo"
-            staging="/var/tmp/arch/autorepo"
-            signPkgs="true"
-            signDB="true"
-            gnupgHome="~/.gnupg"
-            gpgKeyID="0x748231EBCBD808A14F5E85D28C004C2F93481F6B">
-        <!--
-        The mirrors element contains either localMirror elements or remoteMirror elements (see below).
-        There must be at least 1 of either type.
-        -->
-        <mirrors>
-            <!-- localMirror elements contain the path to a local mirror (exists on the same system as you're building
-                 from). Most users will probably want this if their build box and mirror are the same machine, or if
-                 you only want a local repository.
-                    Attributes:
-                        user: The user to chown the files/directories to (must be running as root user). If not
-                              specified, the default is the current user (or the user calling sudo, if done via sudo).
-                        group: The group to chown the files/directories to (must be running as root user). If not
-                               specified, the default is the primary group for the current user (or the user calling
-                               sudo, if done via sudo).
-                        fileMode: The octal permissions to chmod the files to.
-                        dirMode: The octal permissions to chmod the directories to.
-            -->
-            <localMirror
-                    user="foo"
-                    group="bar"
-                    fileMode="0600"
-                    dirMode="0700">/path/to/path</localMirror>
-            <localMirror>a/relative/path</localMirror>
-            <!--
-            The remoteMirror element is for rsyncing packages to a remote mirror/repo server. Rsync must be installed
-            locally (it should; it's part of base-devel) *and* the remote server. Obviously, SSH pubkey auth must also
-            be set up as well for the user. They must have a valid shell on the server for chmodding/chowning.
-                Attributes:
-                    user: The (remote) user to sync as (e.g. for "ssh foo@bar", user would be "foo").
-                    server: The server to sync to. Can be an IP address, hostname (if resolvable), or FQDN.
-                    port: The remote SSH port.
-                    key: The pubkey to use to connect.
-                    remoteUser: The (remote) user to chown the files/directories to (must be connecting as root user).
-                                If not specified, the default is the connecting user ("user" attribute).
-                    remoteGroup: The (remote) group to chown the files/directories to (must be connecting as root user).
-                                 If not specified, the default is the connecting user's ("user" attribute) primary
-                                 group.
-                    fileMode: The octal permissions to chmod the remote files to.
-                    dirMode: The octal permissions to chmod the remote directories to.
-            -->
-            <remoteMirror
-                    user="foo"
-                    server="bar.domain.tld"
-                    port="22"
-                    key="~/.ssh/id_rsa"
-                    remoteUser="foo"
-                    remoteGroup="bar"
-                    fileMode="0600"
-                    dirMode="0700">/path/to/remote/path
-            </remoteMirror>
-        </mirrors>
-        <!--
-        The packages element contains actual packages to build into the repository.
-        -->
-        <packages>
-            <!--
-            The aur element specifies packages that should be fetched and built from the AUR.
-            They contain the name of the package.
-                Attributes:
-                    alwaysBuild: Accepts "1"/"true" or "0"/"false". If true, always build the package even if the same
-                                 version exists already. This only works if you don't delete/empty your staging
-                                 directory, otherwise it will be built.
-            -->
-            <aur alwaysBuild="true">somepkg</aur>
-            <!--
-            The pkgbuild element specifies packages that are locally developed/designed.
-            They contain the name of the package.
-                Attributes:
-                    path: The path to the package to build.
-            -->
-            <pkgbuild path="/path/to/pkgnm.snapshot.tar.gz" alwaysBuild="true">pkgnm</pkgbuild>
-            <pkgbuild path="/path/to/PKGBUILD" alwaysBuild="false">pkgnm2</pkgbuild>
-        </packages>
-    </repo>
-</archrepo>
\ No newline at end of file
diff --git a/arch/autorepo/readme.txt b/arch/autorepo/readme.txt
deleted file mode 100644
index 09b89ba..0000000
--- a/arch/autorepo/readme.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-This has a lot of work pending. I need to factor in configuration files, etc.
-
-But it does require the following packages to be installed, and the buildbox (not the repo mirror server itself) needs to be Arch:
-
-- pacman (duh)
-- namcap
-- devtools (for https://wiki.archlinux.org/index.php/DeveloperWiki:Building_in_a_clean_chroot)
-
-It is designed to be run as a *non-root* user. Use the regen_sudoers.py script to create a sudoers CMND_ALIAS (https://www.sudo.ws/man/1.7.10/sudoers.man.html) to add for your packaging user.
diff --git a/arch/autorepo/regen_sudoers.py b/arch/autorepo/regen_sudoers.py
deleted file mode 100755
index 8dd8454..0000000
--- a/arch/autorepo/regen_sudoers.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-
-sudo_cmds = []
-
-# All of these commands...
-cmds = ['/usr/bin/extra-x86_64-build',
-        '/usr/bin/testing-x86_64-build',
-        '/usr/bin/staging-x86_64-build',
-        '/usr/bin/multilib-build',
-        '/usr/bin/multilib-testing-build',
-        '/usr/bin/multilib-staging-build',
-        '/usr/bin/makechrootpkg']
-
-# Should allow all of these args.
-args = ['-c',
-        '-c -- -- --skippgpcheck --syncdeps --noconfirm --log --holdver --skipinteg',
-        '-- -- --skippgpcheck --syncdeps --noconfirm --log --holdver --skipinteg']
-
-for c in cmds:
-    for a in args:
-        sudo_cmds.append('{0} {1}'.format(c, a))
-
-s = ''
-
-s += 'Cmnd_Alias\tPKGBUILDER = \\\n'
-for c in sudo_cmds:
-    s += '\t\t\t\t{0}, \\\n'.format(c)
-
-s = re.sub(r', \\s*$', '', s)
-print(s)
-
diff --git a/arch/autorepo/sync.sh b/arch/autorepo/sync.sh
deleted file mode 100755
index 32a5537..0000000
--- a/arch/autorepo/sync.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# This obviously will require some tweaking. Will roll into build.py later.
-set -e
-
-server=my_repo.domain.tld
-port=2222
-user=pkgrepo
-src=~/pkgs/built/.
-# You should use rrsync to restrict to a specific directory
-dest='Arch/.'
-
-echo "Syncing..."
-rsync -a --delete -e "ssh -p ${port}" ${src} ${user}@${server}:${dest}
-echo "Done."