r00t2/bootbox
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

Automated commit (/opt/dev/infra/gitclass.py)

Browse Source
This commit is contained in:
brent s. 2020-08-11 23:45:05 -04:00
parent 3dc366595c
commit 6135fb850d
9 changed files with 998 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
builder/__init__.py
View File

@ -1 +1,4 @@
from . import logger
from . import config
from . import compile
from . import ipxe

69
builder/compile.py Normal file
View File

@ -0,0 +1,69 @@
import logging
import os
import shutil
import subprocess
##
from . import constants


_log = logging.getLogger()


class Target(object):
def __init__(self, target_xml):
self.xml = target_xml
self.subdir = self.xml.attrib.get('subDir', '.')
self.base = self.xml.attrib.get('baseName')
self.target = self.xml.text
if not self.base:
self.base = os.path.basename(self.target)


class IpxeScript(object):
def __init__(self, script_dir, script_xml):
self.xml = script_xml
self.fpath = os.path.join(os.path.abspath(os.path.expanduser(script_dir)),
self.xml.text)
self.prefix = self.xml.attrib['prefix']


class Compiler(object):
def __init__(self, builddir, destdir, upstream, patches, monkey_patches, build_xml):
self.xml = build_xml
self.build = os.path.abspath(os.path.expanduser(builddir))
self.dest = os.path.abspath(os.path.expanduser(destdir))
self.src = upstream.dest
self.upstream = upstream
self.patches = patches
self.monkey_patches = monkey_patches
self.targets = []
self.scripts = []
self._add_targets()

def _add_targets(self):
roms = self.xml.findall('rom')
if roms is None:


def make(self):
# NOTE: 1af41000 is the firmware used by virtIO
self.prep()
self.patch()

def patch(self):
for m in self.monkey_patches:
for pf in m.files:
pf.patch()
for p in self.patches:
for pf in p.files:
pf.patch()
return()

def prep(self):
if self.src != self.build:
shutil.copytree(self.src, self.build, dirs_exist_ok = True)
os.makedirs(self.dest)
# These are standard.
for d in constants.IPXE_CATEGORIES:
dpath = os.path.join(self.dest, d)
os.makedirs(dpath, exist_ok = True)

173
builder/config.py Normal file
View File

@ -0,0 +1,173 @@
import copy
import os
import logging
import re
##
import requests
import requests.auth
from lxml import etree
##
from . import constants


_logger = logging.getLogger()


def create_default_cfg():
# Create a stripped sample config.
ws_re = re.compile(r'^\s*$')
samplexml = constants.IPXE_SAMPLE_CONFIG
with open(samplexml, 'rb') as fh:
xml = etree.fromstring(fh.read())
# Create a stripped sample config.
# First we strip comments (and fix the ensuing whitespace).
# etree has a .canonicalize(), but it chokes on a default namespace.
# https://bugs.launchpad.net/lxml/+bug/1869455
# So everything we do is kind of a hack.
# for c in xml.xpath("//comment()"):
# parent = c.getparent()
# parent.remove(c)
xmlstr = etree.tostring(xml, with_comments = False, method = 'c14n', pretty_print = True).decode('utf-8')
newstr = []
for line in xmlstr.splitlines():
r = ws_re.search(line)
if not r:
newstr.append(line.strip())
xml = etree.fromstring(''.join(newstr).encode('utf-8'))
# Remove text and attr text.
xpathq = "descendant-or-self::*[namespace-uri()!='']"
for e in xml.xpath(xpathq):
if e.tag == '{{{0}}}ipxe'.format(xml.nsmap[None]):
continue
if e.text is not None and e.text.strip() != '':
e.text = ''
for k, v in e.attrib.items():
if v is not None:
e.attrib[k] = ''
# Remove multiple children of same type to simplify.
for e in xml.xpath(xpathq):
if e.tag == '{{{0}}}ipxe'.format(xml.nsmap[None]):
continue
parent = e.getparent()
try:
for idx, child in enumerate(parent.findall(e.tag)):
if idx == 0:
continue
parent.remove(child)
except AttributeError:
pass
# And add a comment pointing them to the fully commented config.
xml.insert(0, etree.Comment(('\n Please reference the fully commented example.config.xml found either '
'at:\n '
' * {0}\n * https://git.square-r00t.net/BootBox/tree/builder/'
'example.config.xml\n and then configure this according to those '
'instructions.\n ').format(samplexml)))
return(etree.tostring(xml,
pretty_print = True,
with_comments = True,
with_tail = True,
encoding = 'UTF-8',
xml_declaration = True))


class Config(object):
default_xsd = 'http://schema.xml.r00t2.io/projects/ipxe/build.xsd'
default_xml_path = constants.IPXE_DEFAULT_CFG

def __init__(self, xml_path, *args, **kwargs):
if not xml_path:
xml_path = self.default_xml_path
self.xml_path = os.path.abspath(os.path.expanduser(xml_path))
if not os.path.isfile(self.xml_path):
with open(self.xml_path, 'wb') as fh:
fh.write(create_default_cfg())
_logger.error(('{0} does not exist so a sample configuration file has been created in its place. '
'Be sure to configure it appropriately.').format(self.default_xml_path))
raise ValueError('Config does not exist')
else:
with open(self.xml_path, 'rb') as fh:
self.raw = fh.read()
self.xml = None
self.xsd = None
self.ns_xml = None
self.tree = None
self.ns_tree = None
self.defaults_parser = None
self.parse_xml()
_logger.info('Instantiated {0}.'.format(type(self).__name__))

def get_xsd(self):
raw_xsd = None
base_url = None
xsi = self.xml.nsmap.get('xsi', 'http://www.w3.org/2001/XMLSchema-instance')
schemaLocation = '{{{0}}}schemaLocation'.format(xsi)
schemaURL = self.xml.attrib.get(schemaLocation, self.default_xsd)
split_url = schemaURL.split()
if len(split_url) == 2: # a properly defined schemaLocation
schemaURL = split_url[1]
else:
schemaURL = split_url[0] # a LAZY schemaLocation
if schemaURL.startswith('file://'):
schemaURL = re.sub(r'^file://', r'', schemaURL)
with open(schemaURL, 'rb') as fh:
raw_xsd = fh.read()
base_url = os.path.dirname(schemaURL) + '/'
else:
req = requests.get(schemaURL)
if not req.ok:
raise RuntimeError('Could not download XSD')
raw_xsd = req.content
base_url = os.path.split(req.url)[0] + '/' # This makes me feel dirty.
self.xsd = etree.XMLSchema(etree.XML(raw_xsd, base_url = base_url))
return(None)

def parse_xml(self):
self.parse_raw()
self.get_xsd()
self.populate_defaults()
self.validate()
return(None)

def parse_raw(self, parser = None):
self.xml = etree.fromstring(self.raw, parser = parser)
self.ns_xml = etree.fromstring(self.raw, parser = parser)
self.tree = self.xml.getroottree()
self.ns_tree = self.ns_xml.getroottree()
self.tree.xinclude()
self.ns_tree.xinclude()
self.strip_ns()
return(None)

def populate_defaults(self):
if not self.xsd:
self.get_xsd()
if not self.defaults_parser:
self.defaults_parser = etree.XMLParser(schema = self.xsd, attribute_defaults = True)
self.parse_raw(parser = self.defaults_parser)
return(None)

def remove_defaults(self):
self.parse_raw()
return(None)

def strip_ns(self, obj = None):
# https://stackoverflow.com/questions/30232031/how-can-i-strip-namespaces-out-of-an-lxml-tree/30233635#30233635
xpathq = "descendant-or-self::*[namespace-uri()!='']"
if not obj:
for x in (self.tree, self.xml):
for e in x.xpath(xpathq):
e.tag = etree.QName(e).localname
elif isinstance(obj, (etree._Element, etree._ElementTree)):
obj = copy.deepcopy(obj)
for e in obj.xpath(xpathq):
e.tag = etree.QName(e).localname
return(obj)
else:
raise ValueError('Did not know how to parse obj parameter')
return(None)

def validate(self):
if not self.xsd:
self.get_xsd()
self.xsd.assertValid(self.ns_tree)
return(None)

29
builder/constants.py Normal file
View File

@ -0,0 +1,29 @@
import os

IPXE_GIT_URI = 'git://git.ipxe.org/ipxe.git'
IPXE_ROM_SUFFIXES = ('rom', 'mrom')
IPXE_ROM_FIRMWARES = ('rtl8139',
'8086100e',
'80861209',
'10500940',
'10222000',
'10ec8139',
'1af41000', # firmware used by virtIO
'8086100f',
'808610d3',
'15ad07b0')
_cur_dir = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
IPXE_SAMPLE_CONFIG = os.path.abspath(os.path.join(_cur_dir, 'example.config.xml'))
IPXE_DEFAULT_CFG = '~/.config/bootbox/ipxe_build.xml'
if os.geteuid() == 0:
IPXE_DEFAULT_LOGFILE = '/var/log/bootbox/ipxe.log'
else:
IPXE_DEFAULT_LOGFILE = '~/.cache/bootbox/ipxe.log'
DEF_PORTS = {'git': 9418,
'http': 80,
'https': 443}
IPXE_CATEGORIES = ('default',
'efi',
'iso',
'legacy',
'roms')

72
builder/example.config.xml
View File

@ -2,9 +2,26 @@
<ipxe xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="https://router.r00t2.io/boootbox/ipxe/build/"
xsi:schemaLocation="https://router.r00t2.io/boootbox/ipxe/build/ http://schema.xml.r00t2.io/projects/ipxe/build.xsd">
<!--
Define where the source code should go, where it comes from, and any modifications if desired.
-->
<source srcDir="/opt/builds/ipxe_build">
<upstream>
<git>git://git.ipxe.org/ipxe.git</git>
<!--
Upstream supports two types, "git" (shown below) and "archive". If you use archive, it must be an HTTP file, an
HTTPS file, or a path on the local filesystem (file:///path/to/file.tar.xz). archive supports:
* .tar.bz2/.tbz/.tbz2
* .tar.xz/.txz
* .tar.gz/.tgz
* .tar
* .zip
If you use "git", you must have python-git (https://pypi.org/project/GitPython/) installed.
-->
<git>
<refType>branch</refType>
<ref>master</ref>
<uri>git://git.ipxe.org/ipxe.git</uri>
</git>
<dest>/opt/builds/ipxe_src</dest>
</upstream>
<patchSet>
@ -75,10 +92,57 @@
</opts>
</switchOpts>
<patch patchDir="/opt/builds/patches/ipxe">
<patchFile>git-version.patch</patchFile>
<patchFile>banner.patch</patchFile>
<patchFile>efi-iso.patch</patchFile>
<patchFile stripLevel="1">git-version.patch</patchFile>
<patchFile stripLevel="1">banner.patch</patchFile>
<patchFile stripLevel="1">efi-iso.patch</patchFile>
</patch>
</patchSet>
</source>
<!--
The other attributes enable additional target images. "pxe" is a PXE image, "undi" is an UNDI (RFC4578§2.2)
kernel/NBP, both of which will allow you to chainload iPXE from vanilla PXE.
-->
<build
iso="true"
usb="true"
floppy="true"
mbr="true"
pxe="true"
undi="true">
<!--
If scripts is specified, multiple firmware sets/images will be built along with a non-embedded version.
The required "prefix" attribute is the file prefix that will be appended.
See: https://ipxe.org/scripting
-->
<scripts srcDir="/opt/builds/config/ipxe">
<script prefix="bootstrap">bootstrap.ipxe</script>
<script prefix="">chain-default.ipxe</script>
</scripts>
<!--
The directory where built firmware/images go. They will have respective subdirectories to maintain a clean
hierarchy.
-->
<dest>/opt/builds/built/ipxe</dest>
<!--
The default is all firmware ("make everything").
Note that 1af41000 is the firmware used by VirtIO, and will be symlinked as such if built.
-->
<!--
<roms>
<rom>rtl8139</rom>
<rom>8086100e</rom>
<rom>80861209</rom>
</roms>
-->
<!--
This is optional. If you introduce patches that define additional images that don't follow the above pattern
(such as eworm's EFI ISO patch), you can define them here.
-->
<extraTargets>
<target subDir="efi" baseName="32.efi">bin-i386-efi/ipxe.efi</target>
<target subDir="efi" baseName="64.efi">bin-x86_64-efi/ipxe.efi</target>
<target subDir="iso" baseName="legacy.iso">bin/ipxe.liso</target>
<target subDir="iso" baseName="uefi.iso">bin/ipxe.eiso</target>
</extraTargets>
</build>
</ipxe>

297
builder/ipxe.OLD.py Normal file
View File

@ -0,0 +1,297 @@
#!/usr/bin/env python3

import os
import re
import shutil
import subprocess
##
import git # https://pypi.org/project/GitPython/
import patch_ng # https://pypi.org/project/patch-ng/


patches = ['git-version.patch', 'banner.patch', 'efi-iso.patch']

monkeypatch = {
'config/general.h': {
'enable': [
'NET_PROTO_IPV6',
'DOWNLOAD_PROTO_HTTPS',
'DOWNLOAD_PROTO_FTP',
'DOWNLOAD_PROTO_NFS',
'HTTP_AUTH_NTLM',
# 'HTTP_ENC_PEERDIST',
# 'HTTP_HACK_GCE',
'NSLOOKUP_CMD',
'TIME_CMD',
'DIGEST_CMD',
'LOTEST_CMD',
'VLAN_CMD',
# 'PXE_CMD', # Causes EFI to fail.
'REBOOT_CMD',
'POWEROFF_CMD',
# 'IMAGE_NBI', # I *think* this causes EFI to fail. Can I build it directly as a target w/o enabling?
# 'IMAGE_ELF', # Causes EFI to fail, and is auto-enabled for what needs it.
# 'IMAGE_MULTIBOOT', # Also enabled by default for MBR builds
# 'IMAGE_SCRIPT', # Enabled where needed etc.
# 'IMAGE_BZIMAGE', # http://lists.ipxe.org/pipermail/ipxe-devel/2017-March/005510.html
# 'IMAGE_COMBOOT', # Not really necessary since iPXE has native menus.
# 'IMAGE_EFI', # Enabled by default for EFI builds, and causes build error on bin/ipxe.dsk.tmp
# 'IMAGE_PXE', # EFI builds fail with this. related to PXE_STACK/PXE_CMD?
'IMAGE_TRUST_CMD',
'PCI_CMD',
'PARAM_CMD',
'NEIGHBOUR_CMD',
'PING_CMD',
'CONSOLE_CMD',
'IPSTAT_CMD',
'PROFSTAT_CMD',
'NTP_CMD',
'CERT_CMD'
],
'disable': [
# 'CRYPTO_80211_WEP',
# 'CRYPTO_80211_WPA',
# 'CRYPTO_80211_WPA2',
# 'IWMGMT_CMD'
]},
'config/console.h': {
'enable': [
'CONSOLE_FRAMEBUFFER'
],
'disable': [
# Disables would go here.
]}}

rootdir = '/opt/builds'
logdir = os.path.join(rootdir, 'logs', 'ipxe')
patchdir = os.path.join(rootdir, 'patches', 'ipxe')
buildroot = os.path.join(rootdir, 'ipxe_build')
srcdir = os.path.join(buildroot, 'src')
configdir = os.path.join(rootdir, 'configs', 'ipxe')
destdir = os.path.join(rootdir, 'built', 'ipxe')
repo = git.Repo(buildroot)
master = repo.branches.master
remote = repo.remote('origin')

rom_suffixes = ('rom', 'mrom')
rom_types = ('rtl8139', '8086100e', '80861209', '10500940', '10222000', '10ec8139', '1af41000', '8086100f',
'808610d3', '15ad07b0')


def doMonkeypatch(fname, changeset):
enable_re = None
disable_re = None
fpath = os.path.join(srcdir, fname)
if changeset['enable']:
enable_re = re.compile((r'^\s*(//#define|#undef)\s+'
r'(?P<optname>{0})'
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['enable'])))
if changeset['disable']:
disable_re = re.compile((r'^(#define|//#undef)\s+'
r'(?P<optname>{0})'
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['disable'])))
with open(fpath, 'r') as fh:
configstr = fh.read()
configlines = configstr.splitlines()
for idx, line in enumerate(configlines[:]):
if enable_re:
r = enable_re.search(line)
if r:
configlines[idx] = '#define {0}{1}'.format(r.group('optname'), r.group('comment'))
if disable_re:
r = disable_re.search(line)
if r:
configlines[idx] = '#undef {0}{1}'.format(r.group('optname'), r.group('comment'))
with open(fpath, 'w') as fh:
fh.write('\n'.join(configlines))
fh.write('\n')
return()

def main():
# Cleanup the repo.
repo.head.reset(commit = 'HEAD', working_tree = True)
if repo.active_branch != master:
master.checkout()
repo.head.reset(commit = 'HEAD', working_tree = True)
repo.git.clean('-xdf')
try:
remote.pull()
except BrokenPipeError:
pass
# Patch
for p in patches:
with open(os.path.join(patchdir, p), 'rb') as fh:
patchset = patch_ng.PatchSet(fh)
patchset.apply(strip = 1, root = buildroot)
# "Monkeypatch" - sed-like.
for f, changeset in monkeypatch.items():
doMonkeypatch(f, changeset)
# Build. Finally!
# TODO: ARM support!
# for d in ('default', 'efi', 'iso', 'legacy', 'roms', 'arm'):
for d in ('default', 'efi', 'iso', 'legacy', 'roms'):
dpath = os.path.join(destdir, d)
os.makedirs(dpath, exist_ok = True)
os.makedirs(logdir, exist_ok = True)
os.chdir(srcdir)
## Base files
# TODO: ARM support!
# TODO: efi-sb support (secureboot)!
# http://ipxe.org/appnote/buildtargets
# http://ipxe.org/appnote/buildtargets#special_targets
## BOOTSTRAP ##
with open(os.path.join(logdir, 'all_bootstrap.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'all_bootstrap.stdout'), 'wb') as stdout:
subprocess.run(['make',
'all',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
for fsrc, fdest in (('undionly.kpxe', 'legacy/bootstrap_{f}'),
('ipxe.iso', 'iso/bootstrap_bios.iso'),
('ipxe.usb', 'iso/bootstrap_pxe_usb.img'),
('ipxe.dsk', 'iso/bootstrap_pxe_floppy.img')):
srcpath = os.path.join(srcdir, 'bin', fsrc)
fname = os.path.basename(srcpath)
destpath = os.path.join(destdir, fdest.format(f = fname))
shutil.copy2(srcpath, destpath)
for rom in rom_types:
for s in rom_suffixes:
fname = '{0}.{1}'.format(rom, s)
fpath = os.path.join(srcdir, 'bin', fname)
if os.path.isfile(fpath):
shutil.copy2(fpath,
os.path.join(destdir, 'roms', 'bootstrap_{0}'.format(fname)))
# http://ipxe.org/howto/romburning
# https://libvirt.org/formatdomain.html#elementsNICSROM
if rom == '1af41000':
os.symlink(fpath,
os.path.join(destdir, 'roms', 'bootstrap_virtio.rom'))
## EMBEDDED ##
with open(os.path.join(logdir, 'all.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'all.stdout'), 'wb') as stdout:
subprocess.run(['make',
'all',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
for fsrc, fdest in (('undionly.kpxe', 'legacy/{f}'),
('ipxe.iso', 'iso/bios.iso'),
('ipxe.usb', 'iso/pxe_usb.img'),
('ipxe.dsk', 'iso/pxe_floppy.img')):
srcpath = os.path.join(srcdir, 'bin', fsrc)
fname = os.path.basename(srcpath)
destpath = os.path.join(destdir, fdest.format(f = fname))
shutil.copy2(srcpath, destpath)
for rom in rom_types:
for s in rom_suffixes:
fname = '{0}.{1}'.format(rom, s)
fpath = os.path.join(srcdir, 'bin', fname)
if os.path.isfile(fpath):
shutil.copy2(fpath,
os.path.join(destdir, 'roms', fname))
if rom == '1af41000':
os.symlink(fpath,
os.path.join(destdir, 'roms', 'virtio.rom'))
# DOS/MBR sector/HDD img.
## BOOTSTRAP ##
with open(os.path.join(logdir, 'bootstrap_hdd.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_hdd.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.hd',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
os.path.join(destdir, 'iso', 'bootstrap_legacy_mbr.img'))
## EMBEDDED ##
with open(os.path.join(logdir, 'hdd.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'hdd.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.hd',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
os.path.join(destdir, 'iso', 'legacy_mbr.img'))
# PXE loaders
## BOOTSTRAP ##
with open(os.path.join(logdir, 'bootstrap_loader.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_loader.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.pxe',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
os.path.join(destdir, 'default', 'bootstrap_loader.pxe'))
## EMBEDDED ##
with open(os.path.join(logdir, 'loader.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'loader.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.pxe',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
os.path.join(destdir, 'default', 'loader.pxe'))
# EFI binaries and ISO images
# These have to be done grouped because the eiso stuff APPARENTLY doesn't parse EMBED or lack thereof correctly?
## BOOTSTRAP ##
### EFI ###
with open(os.path.join(logdir, 'bootstrap_efi.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_efi.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin-i386-efi/ipxe.efi',
'bin-x86_64-efi/ipxe.efi',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
os.path.join(destdir, 'efi', 'bootstrap_32.efi'))
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
os.path.join(destdir, 'efi', 'bootstrap_64.efi'))
### UEFI ISO ###
with open(os.path.join(logdir, 'bootstrap_iso.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_iso.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.liso',
'bin/ipxe.eiso',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
os.path.join(destdir, 'iso', 'bootstrap_legacy.iso'))
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
os.path.join(destdir, 'iso', 'bootstrap_uefi.iso'))
## EMBEDDED ##
### EFI ###
with open(os.path.join(logdir, 'efi.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'efi.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin-i386-efi/ipxe.efi',
'bin-x86_64-efi/ipxe.efi',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
os.path.join(destdir, 'efi', '32.efi'))
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
os.path.join(destdir, 'efi', '64.efi'))
### UEFI ISO ###
with open(os.path.join(logdir, 'iso.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'iso.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.liso',
'bin/ipxe.eiso',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
os.path.join(destdir, 'iso', 'legacy.iso'))
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
os.path.join(destdir, 'iso', 'uefi.iso'))
return()

if __name__ == '__main__':
main()

455
builder/ipxe.py
View File

@ -1,297 +1,180 @@
#!/usr/bin/env python3

import io
import os
import logging
import pathlib
import re
import shutil
import subprocess
##
import git # https://pypi.org/project/GitPython/
import patch_ng # https://pypi.org/project/patch-ng/
import patch_ng
##
from . import config
from . import constants
from . import upstream
from . import compile

_log = logging.getLogger()


patches = ['git-version.patch', 'banner.patch', 'efi-iso.patch']
class _Opt(object):
switchtype = None

monkeypatch = {
'config/general.h': {
'enable': [
'NET_PROTO_IPV6',
'DOWNLOAD_PROTO_HTTPS',
'DOWNLOAD_PROTO_FTP',
'DOWNLOAD_PROTO_NFS',
'HTTP_AUTH_NTLM',
# 'HTTP_ENC_PEERDIST',
# 'HTTP_HACK_GCE',
'NSLOOKUP_CMD',
'TIME_CMD',
'DIGEST_CMD',
'LOTEST_CMD',
'VLAN_CMD',
# 'PXE_CMD', # Causes EFI to fail.
'REBOOT_CMD',
'POWEROFF_CMD',
# 'IMAGE_NBI', # I *think* this causes EFI to fail. Can I build it directly as a target w/o enabling?
# 'IMAGE_ELF', # Causes EFI to fail, and is auto-enabled for what needs it.
# 'IMAGE_MULTIBOOT', # Also enabled by default for MBR builds
# 'IMAGE_SCRIPT', # Enabled where needed etc.
# 'IMAGE_BZIMAGE', # http://lists.ipxe.org/pipermail/ipxe-devel/2017-March/005510.html
# 'IMAGE_COMBOOT', # Not really necessary since iPXE has native menus.
# 'IMAGE_EFI', # Enabled by default for EFI builds, and causes build error on bin/ipxe.dsk.tmp
# 'IMAGE_PXE', # EFI builds fail with this. related to PXE_STACK/PXE_CMD?
'IMAGE_TRUST_CMD',
'PCI_CMD',
'PARAM_CMD',
'NEIGHBOUR_CMD',
'PING_CMD',
'CONSOLE_CMD',
'IPSTAT_CMD',
'PROFSTAT_CMD',
'NTP_CMD',
'CERT_CMD'
],
'disable': [
# 'CRYPTO_80211_WEP',
# 'CRYPTO_80211_WPA',
# 'CRYPTO_80211_WPA2',
# 'IWMGMT_CMD'
]},
'config/console.h': {
'enable': [
'CONSOLE_FRAMEBUFFER'
],
'disable': [
# Disables would go here.
]}}

rootdir = '/opt/builds'
logdir = os.path.join(rootdir, 'logs', 'ipxe')
patchdir = os.path.join(rootdir, 'patches', 'ipxe')
buildroot = os.path.join(rootdir, 'ipxe_build')
srcdir = os.path.join(buildroot, 'src')
configdir = os.path.join(rootdir, 'configs', 'ipxe')
destdir = os.path.join(rootdir, 'built', 'ipxe')
repo = git.Repo(buildroot)
master = repo.branches.master
remote = repo.remote('origin')

rom_suffixes = ('rom', 'mrom')
rom_types = ('rtl8139', '8086100e', '80861209', '10500940', '10222000', '10ec8139', '1af41000', '8086100f',
'808610d3', '15ad07b0')
def __init__(self, opt_xml, *args, **kwargs):
self.xml = opt_xml
self.flags = [i.text for i in self.xml.findall('opt')]


def doMonkeypatch(fname, changeset):
enable_re = None
disable_re = None
fpath = os.path.join(srcdir, fname)
if changeset['enable']:
enable_re = re.compile((r'^\s*(//#define|#undef)\s+'
r'(?P<optname>{0})'
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['enable'])))
if changeset['disable']:
disable_re = re.compile((r'^(#define|//#undef)\s+'
r'(?P<optname>{0})'
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['disable'])))
with open(fpath, 'r') as fh:
configstr = fh.read()
configlines = configstr.splitlines()
for idx, line in enumerate(configlines[:]):
if enable_re:
r = enable_re.search(line)
if r:
configlines[idx] = '#define {0}{1}'.format(r.group('optname'), r.group('comment'))
if disable_re:
r = disable_re.search(line)
if r:
configlines[idx] = '#undef {0}{1}'.format(r.group('optname'), r.group('comment'))
with open(fpath, 'w') as fh:
fh.write('\n'.join(configlines))
fh.write('\n')
return()
class _Enable(_Opt):
switchtype = 'enable'

def main():
# Cleanup the repo.
repo.head.reset(commit = 'HEAD', working_tree = True)
if repo.active_branch != master:
master.checkout()
repo.head.reset(commit = 'HEAD', working_tree = True)
repo.git.clean('-xdf')
try:
remote.pull()
except BrokenPipeError:
def __init__(self, opt_xml, *args, **kwargs):
super().__init__(opt_xml, *args, **kwargs)
self.re = re.compile((r'^\s*(//#define|#undef)\s+'
r'(?P<optname>{0})'
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(self.flags)))


class _Disable(_Opt):
switchtype = 'disable'

def __init__(self, opt_xml, *args, **kwargs):
super().__init__(opt_xml, *args, **kwargs)
self.re = re.compile((r'^(#define|//#undef)\s+'
r'(?P<optname>{0})'
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(self.flags)))


class _MonkeyPatchFile(object):
def __init__(self, builddir, opts_xml):
self.xml = opts_xml
self.fpath = os.path.join(os.path.abspath(os.path.expanduser(builddir)),
self.xml.attrib.get('file'))
self.opts = []
if not os.path.isfile(self.fpath):
_log.error('File {0} was due to be monkeypatched but is not found.'.format(self.fpath))
raise FileNotFoundError('File does not exist')
with open(self.fpath, 'r') as fh:
self.buf = io.StringIO(fh.read())
self.buf.seek(0, 0)
self.lines = self.buf.read().splitlines()
self.buf.seek(0, 0)
self._get_opts()

def _get_opts(self):
for opt in self.xml.xpath('./enable|./disable'):
if opt.tag == 'enable':
self.opts.append(_Enable(opt))
else:
self.opts.append(_Disable(opt))
return (None)

def patch(self):
for opt in self.opts:
for idx, line in enumerate(self.lines[:]):
opt_re = opt.re.search(line)
if opt_re:
if opt.switchtype == 'enable':
self.lines[idx] = '#define {0}{1}'.format(opt_re.group('optname'),
opt_re.group('comment'))
else:
self.lines[idx] = '#undef {0}{1}'.format(opt_re.group('optname'),
opt_re.group('comment'))
shutil.copy2(self.fpath, '{0}.orig'.format(self.fpath))
with open(self.fpath, 'w') as fh:
fh.write('\n'.join(self.lines))
fh.write('\n')
return(None)


class MonkeyPatch(object):
def __init__(self, builddir, switchopts_xml):
self.root = os.path.join(os.path.abspath(os.path.expanduser(builddir)),
os.path.abspath(os.path.expanduser(switchopts_xml.attrib.get('subDir', '.'))))
self.xml = switchopts_xml
self.files = []
self._get_files()

def _get_files(self):
for optfile in self.xml.findall('opts'):
self.files.append(_MonkeyPatchFile(self.root, optfile))
return(None)


class _PatchFile(object):
def __init__(self, patchdir, builddir, patchfile_xml):
self.xml = patchfile_xml
self.builddir = os.path.abspath(os.path.expanduser(builddir))
self.fpath = os.path.join(os.path.abspath(os.path.expanduser(patchdir)),
self.xml.text)
self.strip_level = int(self.xml.attrib.get('stripLevel', 1))
if not os.path.isfile(self.fpath):
_log.error('Patch file {0} does not exist'.format(self.fpath))
with open(self.fpath, 'r') as fh:
self.patch_raw = fh.read()
self.patch_obj = patch_ng.PatchSet(self.patch_raw)

def patch(self):
self.patch_obj.apply(strip = self.strip_level, root = self.builddir)
return(None)


class Patch(object):
def __init__(self, builddir, patch_xml):
self.root = os.path.abspath(os.path.expanduser(builddir))
self.xml = patch_xml
_patch_dir = pathlib.Path(self.xml.attrib.get('patchDir', '.'))
if not _patch_dir.is_absolute():
self.patch_dir = os.path.join(self.root, str(_patch_dir))
else:
self.patch_dir = str(_patch_dir)
self.files = []

def _get_patches(self):
for patch in self.xml.findall('patchFile'):
self.files.append(_PatchFile(self.patch_dir, self.root, patch))
return(None)


class Builder(object):
def __init__(self, cfg = None):
if not cfg:
cfg = constants.IPXE_DEFAULT_CFG
self.cfg_file = os.path.abspath(os.path.expanduser(cfg))
self.cfg = config.Config(cfg)
self.xml = self.cfg.xml
self.dest = None
self.builddir = None
self.upstream = None
self.compiler = None
self.monkeypatches = []
self.patches = []
self._get_info()
self._get_patch()
self._get_compile()

def _get_compile(self):
build_xml = self.xml.find('build')
self.compiler = compile.Compiler(self.builddir,
self.dest,
self.upstream,
self.patches,
self.monkeypatches,
build_xml)

def _get_info(self):
source_xml = self.xml.find('source')
build_xml = self.xml.find('build')
self.dest = os.path.abspath(os.path.expanduser(build_xml.find('dest').text))
self.upstream = upstream.upstream_parser(source_xml.find('upstream'))
self.builddir = self.upstream.dest
return(None)

def _get_patch(self):
for patchset in self.xml.xpath('//patchSet'):
for optswitch in patchset.findall('switchOpts'):
self.monkeypatches.append(MonkeyPatch(self.builddir, optswitch))
for patch in self.xml.xpath('//patch'):
self.patches.append(Patch(self.builddir, patch))
return(None)

def build(self):
pass
# Patch
for p in patches:
with open(os.path.join(patchdir, p), 'rb') as fh:
patchset = patch_ng.PatchSet(fh)
patchset.apply(strip = 1, root = buildroot)
# "Monkeypatch" - sed-like.
for f, changeset in monkeypatch.items():
doMonkeypatch(f, changeset)
# Build. Finally!
# TODO: ARM support!
# for d in ('default', 'efi', 'iso', 'legacy', 'roms', 'arm'):
for d in ('default', 'efi', 'iso', 'legacy', 'roms'):
dpath = os.path.join(destdir, d)
os.makedirs(dpath, exist_ok = True)
os.makedirs(logdir, exist_ok = True)
os.chdir(srcdir)
## Base files
# TODO: ARM support!
# TODO: efi-sb support (secureboot)!
# http://ipxe.org/appnote/buildtargets
# http://ipxe.org/appnote/buildtargets#special_targets
## BOOTSTRAP ##
with open(os.path.join(logdir, 'all_bootstrap.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'all_bootstrap.stdout'), 'wb') as stdout:
subprocess.run(['make',
'all',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
for fsrc, fdest in (('undionly.kpxe', 'legacy/bootstrap_{f}'),
('ipxe.iso', 'iso/bootstrap_bios.iso'),
('ipxe.usb', 'iso/bootstrap_pxe_usb.img'),
('ipxe.dsk', 'iso/bootstrap_pxe_floppy.img')):
srcpath = os.path.join(srcdir, 'bin', fsrc)
fname = os.path.basename(srcpath)
destpath = os.path.join(destdir, fdest.format(f = fname))
shutil.copy2(srcpath, destpath)
for rom in rom_types:
for s in rom_suffixes:
fname = '{0}.{1}'.format(rom, s)
fpath = os.path.join(srcdir, 'bin', fname)
if os.path.isfile(fpath):
shutil.copy2(fpath,
os.path.join(destdir, 'roms', 'bootstrap_{0}'.format(fname)))
# http://ipxe.org/howto/romburning
# https://libvirt.org/formatdomain.html#elementsNICSROM
if rom == '1af41000':
os.symlink(fpath,
os.path.join(destdir, 'roms', 'bootstrap_virtio.rom'))
## EMBEDDED ##
with open(os.path.join(logdir, 'all.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'all.stdout'), 'wb') as stdout:
subprocess.run(['make',
'all',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
for fsrc, fdest in (('undionly.kpxe', 'legacy/{f}'),
('ipxe.iso', 'iso/bios.iso'),
('ipxe.usb', 'iso/pxe_usb.img'),
('ipxe.dsk', 'iso/pxe_floppy.img')):
srcpath = os.path.join(srcdir, 'bin', fsrc)
fname = os.path.basename(srcpath)
destpath = os.path.join(destdir, fdest.format(f = fname))
shutil.copy2(srcpath, destpath)
for rom in rom_types:
for s in rom_suffixes:
fname = '{0}.{1}'.format(rom, s)
fpath = os.path.join(srcdir, 'bin', fname)
if os.path.isfile(fpath):
shutil.copy2(fpath,
os.path.join(destdir, 'roms', fname))
if rom == '1af41000':
os.symlink(fpath,
os.path.join(destdir, 'roms', 'virtio.rom'))
# DOS/MBR sector/HDD img.
## BOOTSTRAP ##
with open(os.path.join(logdir, 'bootstrap_hdd.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_hdd.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.hd',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
os.path.join(destdir, 'iso', 'bootstrap_legacy_mbr.img'))
## EMBEDDED ##
with open(os.path.join(logdir, 'hdd.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'hdd.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.hd',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
os.path.join(destdir, 'iso', 'legacy_mbr.img'))
# PXE loaders
## BOOTSTRAP ##
with open(os.path.join(logdir, 'bootstrap_loader.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_loader.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.pxe',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
os.path.join(destdir, 'default', 'bootstrap_loader.pxe'))
## EMBEDDED ##
with open(os.path.join(logdir, 'loader.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'loader.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.pxe',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
os.path.join(destdir, 'default', 'loader.pxe'))
# EFI binaries and ISO images
# These have to be done grouped because the eiso stuff APPARENTLY doesn't parse EMBED or lack thereof correctly?
## BOOTSTRAP ##
### EFI ###
with open(os.path.join(logdir, 'bootstrap_efi.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_efi.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin-i386-efi/ipxe.efi',
'bin-x86_64-efi/ipxe.efi',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
os.path.join(destdir, 'efi', 'bootstrap_32.efi'))
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
os.path.join(destdir, 'efi', 'bootstrap_64.efi'))
### UEFI ISO ###
with open(os.path.join(logdir, 'bootstrap_iso.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'bootstrap_iso.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.liso',
'bin/ipxe.eiso',
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
os.path.join(destdir, 'iso', 'bootstrap_legacy.iso'))
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
os.path.join(destdir, 'iso', 'bootstrap_uefi.iso'))
## EMBEDDED ##
### EFI ###
with open(os.path.join(logdir, 'efi.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'efi.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin-i386-efi/ipxe.efi',
'bin-x86_64-efi/ipxe.efi',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
os.path.join(destdir, 'efi', '32.efi'))
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
os.path.join(destdir, 'efi', '64.efi'))
### UEFI ISO ###
with open(os.path.join(logdir, 'iso.stderr'), 'wb') as stderr, \
open(os.path.join(logdir, 'iso.stdout'), 'wb') as stdout:
subprocess.run(['make',
'bin/ipxe.liso',
'bin/ipxe.eiso',
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
stdout = stdout,
stderr = stderr)
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
os.path.join(destdir, 'iso', 'legacy.iso'))
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
os.path.join(destdir, 'iso', 'uefi.iso'))
return()

if __name__ == '__main__':
main()

56
builder/logger.py Normal file
View File

@ -0,0 +1,56 @@
import logging
import logging.handlers
import os
try:
# https://www.freedesktop.org/software/systemd/python-systemd/journal.html#journalhandler-class
from systemd import journal
_has_journald = True
except ImportError:
_has_journald = False
##
from . import constants


def preplog(logfile = None):
if not logfile:
logfile = constants.IPXE_DEFAULT_LOGFILE
# Prep the log file.
logfile = os.path.abspath(os.path.expanduser(logfile))
os.makedirs(os.path.dirname(logfile), exist_ok = True, mode = 0o0700)
if not os.path.isfile(logfile):
with open(logfile, 'w') as fh:
fh.write('')
os.chmod(logfile, 0o0600)
return(logfile)


# And set up logging.
_cfg_args = {'handlers': [],
'level': logging.DEBUG}
if _has_journald:
# There were some weird changes somewhere along the line.
try:
# But it's *probably* this one.
h = journal.JournalHandler()
except AttributeError:
h = journal.JournaldLogHandler()
# Systemd includes times, so we don't need to.
h.setFormatter(logging.Formatter(style = '{',
fmt = ('{name}:{levelname}:{filename}:'
'{funcName}:{lineno}: {message}')))
_cfg_args['handlers'].append(h)

filehandler = logging.handlers.RotatingFileHandler(preplog(),
encoding = 'utf8',
# Disable rotating for now.
# maxBytes = 50000000000,
# backupCount = 30
)
filehandler.setFormatter(logging.Formatter(style = '{',
fmt = ('{asctime}:'
'{levelname}:{name}:{filename}:'
'{funcName}:{lineno}: {message}')))
_cfg_args['handlers'].append(filehandler)
logging.basicConfig(**_cfg_args)
logger = logging.getLogger('iPXE Builder')
logger.info('Logging initialized.')

134
builder/upstream.py Normal file
View File

@ -0,0 +1,134 @@
import io
import os
import re
import shutil
import tarfile
import zipfile
from urllib.parse import urlparse
##
import requests
try:
import git
_has_git = True
except ImportError:
_has_git = False
##
from . import constants


class _Upstream(object):
def __init__(self, upstream_xml, *args, **kwargs):
self.xml = upstream_xml
self.uri = None
self.proto = None
self.host = None
self.port = None
self.path = None
self.parsed_uri = None
self.dest = os.path.abspath(os.path.expanduser(self.xml.find('dest').text))
os.makedirs(os.path.dirname(self.dest), exist_ok = True, mode = 0o0750)

def parse_uri(self):
self.parsed_uri = urlparse(self.uri)
self.proto = self.parsed_uri.scheme.lower()
self.host = self.parsed_uri.hostname.lower()
self.port = int(getattr(self.parsed_uri, 'port', constants.DEF_PORTS[self.proto]))
self.path = self.parsed_uri.path
return(None)


class Git(_Upstream):
upstreamtype = 'git'

def __init__(self, upstream_xml, refresh = False, *args, **kwargs):
if not _has_git:
raise RuntimeError('The git module (GitPython) is not installed')
super().__init__(upstream_xml, *args, **kwargs)
self.repo = None
self.remote = None
self.ref = None
self.refresh = refresh
git_xml = self.xml.find('git')
try:
self.ref_name = git_xml.find('ref').text
except AttributeError:
self.ref_name = 'master'
try:
self.ref_type = git_xml.find('refType').text
except AttributeError:
self.ref_type = 'branch'
self.uri = git_xml.find('uri').text
self.parse_uri()

def fetch(self):
if os.path.isdir(self.dest) and not self.refresh:
self.repo = git.Repo(self.dest)
elif os.path.isdir(self.dest):
shutil.rmtree(self.dest)
if not self.repo:
self.repo = git.Repo.clone_from(self.uri, self.dest)
for r in self.repo.remotes:
if self.uri in r.urls:
self.remote = r
break
if not self.remote:
self.remote = self.repo.create_remote('bootbox_ipxe_upstream', self.uri)
self.remote.pull()
if self.ref_type in ('branch', 'head'):
try:
self.ref = self.repo.branches[self.ref_name]
except IndexError:
self.ref = self.repo.create_head(self.ref_name, self.remote.refs[self.ref_name])
self.ref.set_tracking_branch(self.ref_name)
self.ref.checkout()
self.repo.head.reset(self.ref, working_tree = True)
elif self.ref_type in ('tag', 'commit', 'rev'):
if self.ref_type == 'tag':
self.ref = self.repo.tags[self.ref_name]
else:
self.ref = self.repo.commit(rev = self.ref_name)
self.repo.head.reset(self.ref, working_tree = True)
self.repo.git.clean('-xdf')
self.remote.pull()
return(None)


class Archive(_Upstream):
upstreamtype = 'archive'

def __init__(self, upstream_xml, *args, **kwargs):
super().__init__(upstream_xml, *args, **kwargs)
self.uri = self.xml.find('archive').text
self.parse_uri()
self.suffix = re.sub(r'^.*\.(?P<suffix>(tar(\.((g|x)z|bz2?))?|t((g|x)z|bz2?)|zip))$',
r'\g<suffix>',
os.path.basename(self.path))
self.req = None
try:
shutil.rmtree(self.dest)
except FileNotFoundError:
pass
os.makedirs(self.dest, mode = 0o0750, exist_ok = True)

def fetch(self):
self.req = requests.get(self.uri)
buf = io.BytesIO(self.req.content)
if not self.req.ok:
raise RuntimeError('There was an error fetching the source')
if self.suffix == 'zip':
handler = zipfile.ZipFile(buf, 'r')
else:
handler = tarfile.open(fileobj = buf, mode = 'r')
handler.extractall(self.dest)
handler.close()
return(None)


def upstream_parser(upstream_xml):
arc = upstream_xml.find('archive')
gitrepo = upstream_xml.find('git')
if arc:
return(Archive(upstream_xml))
elif gitrepo:
return(Git(upstream_xml))
return(None)