r00t2/optools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

committing

Browse Source
This commit is contained in:
brent s 2019-09-18 03:49:52 -04:00
parent 6c7f0a3a6f
commit 62a7d65be5
3 changed files with 264 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
arch/autorepo/autorepo.xsd Normal file
View File

@ -0,0 +1,141 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
targetNamespace="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/tree/"
xmlns:archrepo="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
elementFormDefault="qualified"
attributeFormDefault="unqualified">

<xs:simpleType name="t_posixUserGroup">
<xs:restriction base="xs:token">
<xs:pattern value="[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}$)"/>
<xs:pattern value="%same"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_posixMode">
<xs:restriction base="xs:positiveInteger">
<xs:pattern value="[0-7]?[0-7]{3}"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_path">
<xs:restriction base="xs:string">
<xs:pattern value="(/|~/)?([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_port">
<xs:restriction base="xs:positiveInteger">
<!-- MAN I wish XSD let you validate based on numerical range. -->
<!-- https://stackoverflow.com/a/40213676/733214 -->
<xs:pattern value="([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])"/>
</xs:restriction>
</xs:simpleType>

<xs:complexType name="t_localMirror">
<xs:simpleContent>
<xs:extension base="xs:anyURI">
<xs:attribute name="user" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
<xs:attribute name="group" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
<xs:attribute name="fileMode" type="archrepo:t_posixMode" use="optional" default="0600"/>
<xs:attribute name="dirMode" type="archrepo:t_posixMode" use="optional" default="0700"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>

<xs:complexType name="t_remoteMirror">
<xs:simpleContent>
<xs:extension base="xs:anyURI">
<xs:attribute name="user" type="archrepo:t_posixUserGroup"
default="%same" use="optional"/>
<xs:attribute name="server" type="xs:NMTOKEN" use="required"/>
<xs:attribute name="fileMode" type="archrepo:t_posixMode"
use="optional" default="0600"/>
<xs:attribute name="dirMode" type="archrepo:t_posixMode"
use="optional" default="0700"/>
<xs:attribute name="port" type="archrepo:t_port"
default="22" use="optional"/>
<xs:attribute name="key" type="archrepo:t_path"
default="~/.ssh/id_rsa" use="optional"/>
<xs:attribute name="remoteUser" type="archrepo:t_posixUserGroup"
default="%same" use="optional"/>
<xs:attribute name="remoteGroup" type="archrepo:t_posixUserGroup"
default="%same" use="optional"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>

<xs:simpleType name="t_gpgKeyID">
<xs:restriction base="xs:string">
<xs:pattern value="(0[Xx])?[0-9A-Fa-f]{40}"/>
<xs:pattern value="(0[Xx])?[0-9A-Fa-f]{8}"/>
<xs:pattern value="(0[Xx])?([0-9A-Fa-f]{4} ?){5} *([0-9A-Fa-f]{4}){5}"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:element name="archrepo">
<xs:complexType>
<xs:choice>
<xs:element name="repo" minOccurs="1" maxOccurs="unbounded">
<xs:complexType>
<xs:all minOccurs="1">
<xs:element name="mirrors" minOccurs="1" maxOccurs="1">
<xs:complexType>
<xs:choice minOccurs="1" maxOccurs="unbounded">
<xs:element name="localMirror"
maxOccurs="unbounded"
type="archrepo:t_localMirror"/>
<xs:element name="remoteMirror"
maxOccurs="unbounded"
type="archrepo:t_remoteMirror"/>
</xs:choice>
</xs:complexType>
</xs:element>
<xs:element name="packages" minOccurs="1">
<xs:complexType>
<xs:choice minOccurs="1" maxOccurs="unbounded">
<xs:element name="aur"
maxOccurs="unbounded">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="alwaysBuild" default="true"
type="xs:boolean" use="optional"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
<xs:element name="pkgbuild"
maxOccurs="unbounded">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:token">
<xs:attribute name="path" type="archrepo:t_path"
default="." use="optional"/>
<xs:attribute name="alwaysBuild" default="true"
type="xs:boolean" use="optional"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:choice>
</xs:complexType>
</xs:element>
</xs:all>
<xs:attribute name="name" type="xs:token" use="required"/>
<xs:attribute name="staging" type="archrepo:t_path" use="optional" default="."/>
<xs:attribute name="signPkgs" type="xs:boolean" use="optional" default="true"/>
<xs:attribute name="signDB" type="xs:boolean" use="optional" default="true"/>
<xs:attribute name="gnupgHome" type="archrepo:t_path" use="optional" default="~/.gnupg"/>
<xs:attribute name="gpgKeyID" type="archrepo:t_gpgKeyID" use="optional"/>
</xs:complexType>
</xs:element>
</xs:choice>
</xs:complexType>
</xs:element>
</xs:schema>

40
arch/autorepo/build.py
View File

@ -3,6 +3,7 @@
# TODO: make as flexible as the <rpms>:/bin/build.py (flesh out args), logging, etc. # TODO: make as flexible as the <rpms>:/bin/build.py (flesh out args), logging, etc.


import argparse import argparse
import datetime
import copy import copy
import io import io
import os import os
@ -16,19 +17,18 @@ import warnings
## ##
import gpg import gpg
import requests import requests
from lxml import etree




# TODO: move pkgs to some kind of list/config file/whatever.
# TODO: track which versions are built so we don't need to consistently rebuild ALL packages # TODO: track which versions are built so we don't need to consistently rebuild ALL packages
# You will probably want to change these. # TODO: should this be a configuration option?
_dflts = {'pkgs': ['dumpet'], aurbase = 'https://aur.archlinux.org'
'reponame': 'MY_REPO',
'destdir': '~/pkgs/built', _dflts = {'cfgfile': '~/.config/optools/arch/autorepo.xml'}
'aurbase': 'https://aur.archlinux.org'}




class Packager(object): class Packager(object):
def __init__(self, *args, **kwargs): def __init__(self, cfgfile = _dflts['cfgfile'], *args, **kwargs):
user_params = kwargs user_params = kwargs
self.args = copy.deepcopy(_dflts) self.args = copy.deepcopy(_dflts)
self.args.update(user_params) self.args.update(user_params)
@ -173,28 +173,10 @@ class Packager(object):


def parseArgs(): def parseArgs():
args = argparse.ArgumentParser(description = 'Build Pacman packages and update a local repository') args = argparse.ArgumentParser(description = 'Build Pacman packages and update a local repository')
args.add_argument('-p', '--package', args.add_argument('-c', '--config',
dest = 'pkgs', dest = 'cfgfile',
action = 'append', default = _dflts['cfgfile'],
help = ('If specified, only build for this package name. Can be specified multiple times. ' help = ('The path to the configuration file. Default: {0}').format(_dflts['cfgfile']))
'(Default is hardcoded: {0})').format(', '.join(_dflts['pkgs'])))
args.add_argument('-r', '--repo-name',
dest = 'reponame',
default = _dflts['reponame'],
help = ('The name of the repo. Default: {0}').format(_dflts['reponame']))
args.add_argument('-d', '--dest-dir',
dest = 'destdir',
default = _dflts['destdir'],
help = ('Where the built packages should go. Default: {0}').format(_dflts['destdir']))
args.add_argument('-a', '--aur-base',
dest = 'aurbase',
default = _dflts['aurbase'],
help = ('The base URL for AUR. You probably don\'t want to change this. '
'Default: {0}').format(_dflts['aurbase']))
args.add_argument('-A', '--aur-only',
dest = 'auronly',
action = 'store_true',
help = ('If specified, ignore local PKGBUILDs and only build from AUR'))
return(args) return(args)


def main(): def main():

112
arch/autorepo/example.pkgs.xml Normal file
View File

@ -0,0 +1,112 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
The parsing supports XInclude (https://www.w3.org/TR/xinclude/).
You can use external XML snippets if that's easier/cleaner (it usually is).
-->
<archrepo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/arch/autorepo/autorepo.xsd">
<!--
The repo element contains information for each repository we should build for.
Attributes:
name: The name of the repository. This is used for the db name and to generate pacman.conf snippets.
staging: The path to the staging directory. This is where we will build packages and sync to mirrors from.
signPkgs: Either "1"/"true" or "0"/"false". Whether or not we should sign packages. See signDB, gnupgHome,
and gpgKeyID.
signDB: Either "1"/"true" or "0"/"false". Whether or not we should sign the database files. See signPkgs,
gnupgHome, and gpgKeyID.
gnupgHome: The path to use for the GnuPG home (GNUPGHOME environment variable).
The order of preference follows:
1.) gnupgHome attribute (if set)
2.) $GNUPGHOME env var (if set)
3.) ~/.gnupg
See signPkgs, signDB, and gpgKeyID.
gpgKeyID: The key ID to use. It *must* have the signing ("S") capability. If it is a subkey fingerprint,
that subkey will be used. If a subkey fpr is specified but lacks the signing capability, the
(parent) key will be used (if it has signing capability). If no key ID/fingerprint/etc. is
specified, we will use the first key with signing capability found (this should be fine if you
only have one key with signing capabilities in your gnupgHome). If no suitable key is found but
signing is enabled, an error will be thrown. See signPkgs, signDB, and gnupgHome.
-->
<repo
name="testrepo"
staging="/var/tmp/arch/autorepo"
signPkgs="true"
signDB="true"
gnupgHome="~/.gnupg"
gpgKeyID="0x748231EBCBD808A14F5E85D28C004C2F93481F6B">
<!--
The mirrors element contains either localMirror elements or remoteMirror elements (see below).
There must be at least 1 of either type.
-->
<mirrors>
<!-- localMirror elements contain the path to a local mirror (exists on the same system as you're building
from). Most users will probably want this if their build box and mirror are the same machine, or if
you only want a local repository.
Attributes:
user: The user to chown the files/directories to (must be running as root user). If not
specified, the default is the current user (or the user calling sudo, if done via sudo).
group: The group to chown the files/directories to (must be running as root user). If not
specified, the default is the primary group for the current user (or the user calling
sudo, if done via sudo).
fileMode: The octal permissions to chmod the files to.
dirMode: The octal permissions to chmod the directories to.
-->
<localMirror
user="foo"
group="bar"
fileMode="0600"
dirMode="0700">/path/to/path</localMirror>
<localMirror>a/relative/path</localMirror>
<!--
The remoteMirror element is for rsyncing packages to a remote mirror/repo server. Rsync must be installed
locally (it should; it's part of base-devel) *and* the remote server. Obviously, SSH pubkey auth must also
be set up as well for the user. They must have a valid shell on the server for chmodding/chowning.
Attributes:
user: The (remote) user to sync as (e.g. for "ssh foo@bar", user would be "foo").
server: The server to sync to. Can be an IP address, hostname (if resolvable), or FQDN.
port: The remote SSH port.
key: The pubkey to use to connect.
remoteUser: The (remote) user to chown the files/directories to (must be connecting as root user).
If not specified, the default is the connecting user ("user" attribute).
remoteGroup: The (remote) group to chown the files/directories to (must be connecting as root user).
If not specified, the default is the connecting user's ("user" attribute) primary
group.
fileMode: The octal permissions to chmod the remote files to.
dirMode: The octal permissions to chmod the remote directories to.
-->
<remoteMirror
user="foo"
server="bar.domain.tld"
port="22"
key="~/.ssh/id_rsa"
remoteUser="foo"
remoteGroup="bar"
fileMode="0600"
dirMode="0700">/path/to/remote/path
</remoteMirror>
</mirrors>
<!--
The packages element contains actual packages to build into the repository.
-->
<packages>
<!--
The aur element specifies packages that should be fetched and built from the AUR.
They contain the name of the package.
Attributes:
alwaysBuild: Accepts "1"/"true" or "0"/"false". If true, always build the package even if the same
version exists already. This only works if you don't delete/empty your staging
directory, otherwise it will be built.
-->
<aur alwaysBuild="true">somepkg</aur>
<!--
The pkgbuild element specifies packages that are locally developed/designed.
They contain the name of the package.
Attributes:
path: The path to the package to build.
-->
<pkgbuild path="/path/to/pkgnm.snapshot.tar.gz" alwaysBuild="true">pkgnm</pkgbuild>
<pkgbuild path="/path/to/PKGBUILD" alwaysBuild="false">pkgnm2</pkgbuild>
</packages>
</repo>
</archrepo>