diff --git a/builder/__init__.py b/builder/__init__.py index 81a8cd0..1c0c11d 100644 --- a/builder/__init__.py +++ b/builder/__init__.py @@ -1 +1,4 @@ +from . import logger +from . import config +from . import compile from . import ipxe diff --git a/builder/compile.py b/builder/compile.py new file mode 100644 index 0000000..963bb84 --- /dev/null +++ b/builder/compile.py @@ -0,0 +1,69 @@ +import logging +import os +import shutil +import subprocess +## +from . import constants + + +_log = logging.getLogger() + + +class Target(object): + def __init__(self, target_xml): + self.xml = target_xml + self.subdir = self.xml.attrib.get('subDir', '.') + self.base = self.xml.attrib.get('baseName') + self.target = self.xml.text + if not self.base: + self.base = os.path.basename(self.target) + + +class IpxeScript(object): + def __init__(self, script_dir, script_xml): + self.xml = script_xml + self.fpath = os.path.join(os.path.abspath(os.path.expanduser(script_dir)), + self.xml.text) + self.prefix = self.xml.attrib['prefix'] + + +class Compiler(object): + def __init__(self, builddir, destdir, upstream, patches, monkey_patches, build_xml): + self.xml = build_xml + self.build = os.path.abspath(os.path.expanduser(builddir)) + self.dest = os.path.abspath(os.path.expanduser(destdir)) + self.src = upstream.dest + self.upstream = upstream + self.patches = patches + self.monkey_patches = monkey_patches + self.targets = [] + self.scripts = [] + self._add_targets() + + def _add_targets(self): + roms = self.xml.findall('rom') + if roms is None: + + + def make(self): + # NOTE: 1af41000 is the firmware used by virtIO + self.prep() + self.patch() + + def patch(self): + for m in self.monkey_patches: + for pf in m.files: + pf.patch() + for p in self.patches: + for pf in p.files: + pf.patch() + return() + + def prep(self): + if self.src != self.build: + shutil.copytree(self.src, self.build, dirs_exist_ok = True) + os.makedirs(self.dest) + # These are standard. + for d in constants.IPXE_CATEGORIES: + dpath = os.path.join(self.dest, d) + os.makedirs(dpath, exist_ok = True) diff --git a/builder/config.py b/builder/config.py new file mode 100644 index 0000000..6ea5231 --- /dev/null +++ b/builder/config.py @@ -0,0 +1,173 @@ +import copy +import os +import logging +import re +## +import requests +import requests.auth +from lxml import etree +## +from . import constants + + +_logger = logging.getLogger() + + +def create_default_cfg(): + # Create a stripped sample config. + ws_re = re.compile(r'^\s*$') + samplexml = constants.IPXE_SAMPLE_CONFIG + with open(samplexml, 'rb') as fh: + xml = etree.fromstring(fh.read()) + # Create a stripped sample config. + # First we strip comments (and fix the ensuing whitespace). + # etree has a .canonicalize(), but it chokes on a default namespace. + # https://bugs.launchpad.net/lxml/+bug/1869455 + # So everything we do is kind of a hack. + # for c in xml.xpath("//comment()"): + # parent = c.getparent() + # parent.remove(c) + xmlstr = etree.tostring(xml, with_comments = False, method = 'c14n', pretty_print = True).decode('utf-8') + newstr = [] + for line in xmlstr.splitlines(): + r = ws_re.search(line) + if not r: + newstr.append(line.strip()) + xml = etree.fromstring(''.join(newstr).encode('utf-8')) + # Remove text and attr text. + xpathq = "descendant-or-self::*[namespace-uri()!='']" + for e in xml.xpath(xpathq): + if e.tag == '{{{0}}}ipxe'.format(xml.nsmap[None]): + continue + if e.text is not None and e.text.strip() != '': + e.text = '' + for k, v in e.attrib.items(): + if v is not None: + e.attrib[k] = '' + # Remove multiple children of same type to simplify. + for e in xml.xpath(xpathq): + if e.tag == '{{{0}}}ipxe'.format(xml.nsmap[None]): + continue + parent = e.getparent() + try: + for idx, child in enumerate(parent.findall(e.tag)): + if idx == 0: + continue + parent.remove(child) + except AttributeError: + pass + # And add a comment pointing them to the fully commented config. + xml.insert(0, etree.Comment(('\n Please reference the fully commented example.config.xml found either ' + 'at:\n ' + ' * {0}\n * https://git.square-r00t.net/BootBox/tree/builder/' + 'example.config.xml\n and then configure this according to those ' + 'instructions.\n ').format(samplexml))) + return(etree.tostring(xml, + pretty_print = True, + with_comments = True, + with_tail = True, + encoding = 'UTF-8', + xml_declaration = True)) + + +class Config(object): + default_xsd = 'http://schema.xml.r00t2.io/projects/ipxe/build.xsd' + default_xml_path = constants.IPXE_DEFAULT_CFG + + def __init__(self, xml_path, *args, **kwargs): + if not xml_path: + xml_path = self.default_xml_path + self.xml_path = os.path.abspath(os.path.expanduser(xml_path)) + if not os.path.isfile(self.xml_path): + with open(self.xml_path, 'wb') as fh: + fh.write(create_default_cfg()) + _logger.error(('{0} does not exist so a sample configuration file has been created in its place. ' + 'Be sure to configure it appropriately.').format(self.default_xml_path)) + raise ValueError('Config does not exist') + else: + with open(self.xml_path, 'rb') as fh: + self.raw = fh.read() + self.xml = None + self.xsd = None + self.ns_xml = None + self.tree = None + self.ns_tree = None + self.defaults_parser = None + self.parse_xml() + _logger.info('Instantiated {0}.'.format(type(self).__name__)) + + def get_xsd(self): + raw_xsd = None + base_url = None + xsi = self.xml.nsmap.get('xsi', 'http://www.w3.org/2001/XMLSchema-instance') + schemaLocation = '{{{0}}}schemaLocation'.format(xsi) + schemaURL = self.xml.attrib.get(schemaLocation, self.default_xsd) + split_url = schemaURL.split() + if len(split_url) == 2: # a properly defined schemaLocation + schemaURL = split_url[1] + else: + schemaURL = split_url[0] # a LAZY schemaLocation + if schemaURL.startswith('file://'): + schemaURL = re.sub(r'^file://', r'', schemaURL) + with open(schemaURL, 'rb') as fh: + raw_xsd = fh.read() + base_url = os.path.dirname(schemaURL) + '/' + else: + req = requests.get(schemaURL) + if not req.ok: + raise RuntimeError('Could not download XSD') + raw_xsd = req.content + base_url = os.path.split(req.url)[0] + '/' # This makes me feel dirty. + self.xsd = etree.XMLSchema(etree.XML(raw_xsd, base_url = base_url)) + return(None) + + def parse_xml(self): + self.parse_raw() + self.get_xsd() + self.populate_defaults() + self.validate() + return(None) + + def parse_raw(self, parser = None): + self.xml = etree.fromstring(self.raw, parser = parser) + self.ns_xml = etree.fromstring(self.raw, parser = parser) + self.tree = self.xml.getroottree() + self.ns_tree = self.ns_xml.getroottree() + self.tree.xinclude() + self.ns_tree.xinclude() + self.strip_ns() + return(None) + + def populate_defaults(self): + if not self.xsd: + self.get_xsd() + if not self.defaults_parser: + self.defaults_parser = etree.XMLParser(schema = self.xsd, attribute_defaults = True) + self.parse_raw(parser = self.defaults_parser) + return(None) + + def remove_defaults(self): + self.parse_raw() + return(None) + + def strip_ns(self, obj = None): + # https://stackoverflow.com/questions/30232031/how-can-i-strip-namespaces-out-of-an-lxml-tree/30233635#30233635 + xpathq = "descendant-or-self::*[namespace-uri()!='']" + if not obj: + for x in (self.tree, self.xml): + for e in x.xpath(xpathq): + e.tag = etree.QName(e).localname + elif isinstance(obj, (etree._Element, etree._ElementTree)): + obj = copy.deepcopy(obj) + for e in obj.xpath(xpathq): + e.tag = etree.QName(e).localname + return(obj) + else: + raise ValueError('Did not know how to parse obj parameter') + return(None) + + def validate(self): + if not self.xsd: + self.get_xsd() + self.xsd.assertValid(self.ns_tree) + return(None) diff --git a/builder/constants.py b/builder/constants.py new file mode 100644 index 0000000..28016d5 --- /dev/null +++ b/builder/constants.py @@ -0,0 +1,29 @@ +import os + +IPXE_GIT_URI = 'git://git.ipxe.org/ipxe.git' +IPXE_ROM_SUFFIXES = ('rom', 'mrom') +IPXE_ROM_FIRMWARES = ('rtl8139', + '8086100e', + '80861209', + '10500940', + '10222000', + '10ec8139', + '1af41000', # firmware used by virtIO + '8086100f', + '808610d3', + '15ad07b0') +_cur_dir = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +IPXE_SAMPLE_CONFIG = os.path.abspath(os.path.join(_cur_dir, 'example.config.xml')) +IPXE_DEFAULT_CFG = '~/.config/bootbox/ipxe_build.xml' +if os.geteuid() == 0: + IPXE_DEFAULT_LOGFILE = '/var/log/bootbox/ipxe.log' +else: + IPXE_DEFAULT_LOGFILE = '~/.cache/bootbox/ipxe.log' +DEF_PORTS = {'git': 9418, + 'http': 80, + 'https': 443} +IPXE_CATEGORIES = ('default', + 'efi', + 'iso', + 'legacy', + 'roms') diff --git a/builder/example.config.xml b/builder/example.config.xml index b100441..cbe7afd 100644 --- a/builder/example.config.xml +++ b/builder/example.config.xml @@ -2,9 +2,26 @@ + - git://git.ipxe.org/ipxe.git + + + branch + master + git://git.ipxe.org/ipxe.git + /opt/builds/ipxe_src @@ -75,10 +92,57 @@ - git-version.patch - banner.patch - efi-iso.patch + git-version.patch + banner.patch + efi-iso.patch + + + + + + + + + /opt/builds/built/ipxe + + + + + bin-i386-efi/ipxe.efi + bin-x86_64-efi/ipxe.efi + bin/ipxe.liso + bin/ipxe.eiso + + diff --git a/builder/ipxe.OLD.py b/builder/ipxe.OLD.py new file mode 100644 index 0000000..2dc1695 --- /dev/null +++ b/builder/ipxe.OLD.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 + +import os +import re +import shutil +import subprocess +## +import git # https://pypi.org/project/GitPython/ +import patch_ng # https://pypi.org/project/patch-ng/ + + +patches = ['git-version.patch', 'banner.patch', 'efi-iso.patch'] + +monkeypatch = { + 'config/general.h': { + 'enable': [ + 'NET_PROTO_IPV6', + 'DOWNLOAD_PROTO_HTTPS', + 'DOWNLOAD_PROTO_FTP', + 'DOWNLOAD_PROTO_NFS', + 'HTTP_AUTH_NTLM', + # 'HTTP_ENC_PEERDIST', + # 'HTTP_HACK_GCE', + 'NSLOOKUP_CMD', + 'TIME_CMD', + 'DIGEST_CMD', + 'LOTEST_CMD', + 'VLAN_CMD', + # 'PXE_CMD', # Causes EFI to fail. + 'REBOOT_CMD', + 'POWEROFF_CMD', + # 'IMAGE_NBI', # I *think* this causes EFI to fail. Can I build it directly as a target w/o enabling? + # 'IMAGE_ELF', # Causes EFI to fail, and is auto-enabled for what needs it. + # 'IMAGE_MULTIBOOT', # Also enabled by default for MBR builds + # 'IMAGE_SCRIPT', # Enabled where needed etc. + # 'IMAGE_BZIMAGE', # http://lists.ipxe.org/pipermail/ipxe-devel/2017-March/005510.html + # 'IMAGE_COMBOOT', # Not really necessary since iPXE has native menus. + # 'IMAGE_EFI', # Enabled by default for EFI builds, and causes build error on bin/ipxe.dsk.tmp + # 'IMAGE_PXE', # EFI builds fail with this. related to PXE_STACK/PXE_CMD? + 'IMAGE_TRUST_CMD', + 'PCI_CMD', + 'PARAM_CMD', + 'NEIGHBOUR_CMD', + 'PING_CMD', + 'CONSOLE_CMD', + 'IPSTAT_CMD', + 'PROFSTAT_CMD', + 'NTP_CMD', + 'CERT_CMD' + ], + 'disable': [ + # 'CRYPTO_80211_WEP', + # 'CRYPTO_80211_WPA', + # 'CRYPTO_80211_WPA2', + # 'IWMGMT_CMD' + ]}, + 'config/console.h': { + 'enable': [ + 'CONSOLE_FRAMEBUFFER' + ], + 'disable': [ + # Disables would go here. + ]}} + +rootdir = '/opt/builds' +logdir = os.path.join(rootdir, 'logs', 'ipxe') +patchdir = os.path.join(rootdir, 'patches', 'ipxe') +buildroot = os.path.join(rootdir, 'ipxe_build') +srcdir = os.path.join(buildroot, 'src') +configdir = os.path.join(rootdir, 'configs', 'ipxe') +destdir = os.path.join(rootdir, 'built', 'ipxe') +repo = git.Repo(buildroot) +master = repo.branches.master +remote = repo.remote('origin') + +rom_suffixes = ('rom', 'mrom') +rom_types = ('rtl8139', '8086100e', '80861209', '10500940', '10222000', '10ec8139', '1af41000', '8086100f', + '808610d3', '15ad07b0') + + +def doMonkeypatch(fname, changeset): + enable_re = None + disable_re = None + fpath = os.path.join(srcdir, fname) + if changeset['enable']: + enable_re = re.compile((r'^\s*(//#define|#undef)\s+' + r'(?P{0})' + r'(?P\s+/\*.*)?\s*$').format('|'.join(changeset['enable']))) + if changeset['disable']: + disable_re = re.compile((r'^(#define|//#undef)\s+' + r'(?P{0})' + r'(?P\s+/\*.*)?\s*$').format('|'.join(changeset['disable']))) + with open(fpath, 'r') as fh: + configstr = fh.read() + configlines = configstr.splitlines() + for idx, line in enumerate(configlines[:]): + if enable_re: + r = enable_re.search(line) + if r: + configlines[idx] = '#define {0}{1}'.format(r.group('optname'), r.group('comment')) + if disable_re: + r = disable_re.search(line) + if r: + configlines[idx] = '#undef {0}{1}'.format(r.group('optname'), r.group('comment')) + with open(fpath, 'w') as fh: + fh.write('\n'.join(configlines)) + fh.write('\n') + return() + +def main(): + # Cleanup the repo. + repo.head.reset(commit = 'HEAD', working_tree = True) + if repo.active_branch != master: + master.checkout() + repo.head.reset(commit = 'HEAD', working_tree = True) + repo.git.clean('-xdf') + try: + remote.pull() + except BrokenPipeError: + pass + # Patch + for p in patches: + with open(os.path.join(patchdir, p), 'rb') as fh: + patchset = patch_ng.PatchSet(fh) + patchset.apply(strip = 1, root = buildroot) + # "Monkeypatch" - sed-like. + for f, changeset in monkeypatch.items(): + doMonkeypatch(f, changeset) + # Build. Finally! + # TODO: ARM support! + # for d in ('default', 'efi', 'iso', 'legacy', 'roms', 'arm'): + for d in ('default', 'efi', 'iso', 'legacy', 'roms'): + dpath = os.path.join(destdir, d) + os.makedirs(dpath, exist_ok = True) + os.makedirs(logdir, exist_ok = True) + os.chdir(srcdir) + ## Base files + # TODO: ARM support! + # TODO: efi-sb support (secureboot)! + # http://ipxe.org/appnote/buildtargets + # http://ipxe.org/appnote/buildtargets#special_targets + ## BOOTSTRAP ## + with open(os.path.join(logdir, 'all_bootstrap.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'all_bootstrap.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'all', + 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], + stdout = stdout, + stderr = stderr) + for fsrc, fdest in (('undionly.kpxe', 'legacy/bootstrap_{f}'), + ('ipxe.iso', 'iso/bootstrap_bios.iso'), + ('ipxe.usb', 'iso/bootstrap_pxe_usb.img'), + ('ipxe.dsk', 'iso/bootstrap_pxe_floppy.img')): + srcpath = os.path.join(srcdir, 'bin', fsrc) + fname = os.path.basename(srcpath) + destpath = os.path.join(destdir, fdest.format(f = fname)) + shutil.copy2(srcpath, destpath) + for rom in rom_types: + for s in rom_suffixes: + fname = '{0}.{1}'.format(rom, s) + fpath = os.path.join(srcdir, 'bin', fname) + if os.path.isfile(fpath): + shutil.copy2(fpath, + os.path.join(destdir, 'roms', 'bootstrap_{0}'.format(fname))) + # http://ipxe.org/howto/romburning + # https://libvirt.org/formatdomain.html#elementsNICSROM + if rom == '1af41000': + os.symlink(fpath, + os.path.join(destdir, 'roms', 'bootstrap_virtio.rom')) + ## EMBEDDED ## + with open(os.path.join(logdir, 'all.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'all.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'all', + 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], + stdout = stdout, + stderr = stderr) + for fsrc, fdest in (('undionly.kpxe', 'legacy/{f}'), + ('ipxe.iso', 'iso/bios.iso'), + ('ipxe.usb', 'iso/pxe_usb.img'), + ('ipxe.dsk', 'iso/pxe_floppy.img')): + srcpath = os.path.join(srcdir, 'bin', fsrc) + fname = os.path.basename(srcpath) + destpath = os.path.join(destdir, fdest.format(f = fname)) + shutil.copy2(srcpath, destpath) + for rom in rom_types: + for s in rom_suffixes: + fname = '{0}.{1}'.format(rom, s) + fpath = os.path.join(srcdir, 'bin', fname) + if os.path.isfile(fpath): + shutil.copy2(fpath, + os.path.join(destdir, 'roms', fname)) + if rom == '1af41000': + os.symlink(fpath, + os.path.join(destdir, 'roms', 'virtio.rom')) + # DOS/MBR sector/HDD img. + ## BOOTSTRAP ## + with open(os.path.join(logdir, 'bootstrap_hdd.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'bootstrap_hdd.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin/ipxe.hd', + 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], + stdout = stdout, + stderr = stderr) + shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'), + os.path.join(destdir, 'iso', 'bootstrap_legacy_mbr.img')) + ## EMBEDDED ## + with open(os.path.join(logdir, 'hdd.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'hdd.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin/ipxe.hd', + 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], + stdout = stdout, + stderr = stderr) + shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'), + os.path.join(destdir, 'iso', 'legacy_mbr.img')) + # PXE loaders + ## BOOTSTRAP ## + with open(os.path.join(logdir, 'bootstrap_loader.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'bootstrap_loader.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin/ipxe.pxe', + 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], + stdout = stdout, + stderr = stderr) + os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'), + os.path.join(destdir, 'default', 'bootstrap_loader.pxe')) + ## EMBEDDED ## + with open(os.path.join(logdir, 'loader.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'loader.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin/ipxe.pxe', + 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], + stdout = stdout, + stderr = stderr) + os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'), + os.path.join(destdir, 'default', 'loader.pxe')) + # EFI binaries and ISO images + # These have to be done grouped because the eiso stuff APPARENTLY doesn't parse EMBED or lack thereof correctly? + ## BOOTSTRAP ## + ### EFI ### + with open(os.path.join(logdir, 'bootstrap_efi.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'bootstrap_efi.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin-i386-efi/ipxe.efi', + 'bin-x86_64-efi/ipxe.efi', + 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], + stdout = stdout, + stderr = stderr) + shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'), + os.path.join(destdir, 'efi', 'bootstrap_32.efi')) + shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'), + os.path.join(destdir, 'efi', 'bootstrap_64.efi')) + ### UEFI ISO ### + with open(os.path.join(logdir, 'bootstrap_iso.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'bootstrap_iso.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin/ipxe.liso', + 'bin/ipxe.eiso', + 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], + stdout = stdout, + stderr = stderr) + os.rename(os.path.join(srcdir, 'bin/ipxe.liso'), + os.path.join(destdir, 'iso', 'bootstrap_legacy.iso')) + os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'), + os.path.join(destdir, 'iso', 'bootstrap_uefi.iso')) + ## EMBEDDED ## + ### EFI ### + with open(os.path.join(logdir, 'efi.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'efi.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin-i386-efi/ipxe.efi', + 'bin-x86_64-efi/ipxe.efi', + 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], + stdout = stdout, + stderr = stderr) + shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'), + os.path.join(destdir, 'efi', '32.efi')) + shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'), + os.path.join(destdir, 'efi', '64.efi')) + ### UEFI ISO ### + with open(os.path.join(logdir, 'iso.stderr'), 'wb') as stderr, \ + open(os.path.join(logdir, 'iso.stdout'), 'wb') as stdout: + subprocess.run(['make', + 'bin/ipxe.liso', + 'bin/ipxe.eiso', + 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], + stdout = stdout, + stderr = stderr) + os.rename(os.path.join(srcdir, 'bin/ipxe.liso'), + os.path.join(destdir, 'iso', 'legacy.iso')) + os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'), + os.path.join(destdir, 'iso', 'uefi.iso')) + return() + +if __name__ == '__main__': + main() diff --git a/builder/ipxe.py b/builder/ipxe.py index 2dc1695..44bdacb 100644 --- a/builder/ipxe.py +++ b/builder/ipxe.py @@ -1,297 +1,180 @@ -#!/usr/bin/env python3 - +import io import os +import logging +import pathlib import re import shutil -import subprocess ## -import git # https://pypi.org/project/GitPython/ -import patch_ng # https://pypi.org/project/patch-ng/ +import patch_ng +## +from . import config +from . import constants +from . import upstream +from . import compile + +_log = logging.getLogger() -patches = ['git-version.patch', 'banner.patch', 'efi-iso.patch'] +class _Opt(object): + switchtype = None -monkeypatch = { - 'config/general.h': { - 'enable': [ - 'NET_PROTO_IPV6', - 'DOWNLOAD_PROTO_HTTPS', - 'DOWNLOAD_PROTO_FTP', - 'DOWNLOAD_PROTO_NFS', - 'HTTP_AUTH_NTLM', - # 'HTTP_ENC_PEERDIST', - # 'HTTP_HACK_GCE', - 'NSLOOKUP_CMD', - 'TIME_CMD', - 'DIGEST_CMD', - 'LOTEST_CMD', - 'VLAN_CMD', - # 'PXE_CMD', # Causes EFI to fail. - 'REBOOT_CMD', - 'POWEROFF_CMD', - # 'IMAGE_NBI', # I *think* this causes EFI to fail. Can I build it directly as a target w/o enabling? - # 'IMAGE_ELF', # Causes EFI to fail, and is auto-enabled for what needs it. - # 'IMAGE_MULTIBOOT', # Also enabled by default for MBR builds - # 'IMAGE_SCRIPT', # Enabled where needed etc. - # 'IMAGE_BZIMAGE', # http://lists.ipxe.org/pipermail/ipxe-devel/2017-March/005510.html - # 'IMAGE_COMBOOT', # Not really necessary since iPXE has native menus. - # 'IMAGE_EFI', # Enabled by default for EFI builds, and causes build error on bin/ipxe.dsk.tmp - # 'IMAGE_PXE', # EFI builds fail with this. related to PXE_STACK/PXE_CMD? - 'IMAGE_TRUST_CMD', - 'PCI_CMD', - 'PARAM_CMD', - 'NEIGHBOUR_CMD', - 'PING_CMD', - 'CONSOLE_CMD', - 'IPSTAT_CMD', - 'PROFSTAT_CMD', - 'NTP_CMD', - 'CERT_CMD' - ], - 'disable': [ - # 'CRYPTO_80211_WEP', - # 'CRYPTO_80211_WPA', - # 'CRYPTO_80211_WPA2', - # 'IWMGMT_CMD' - ]}, - 'config/console.h': { - 'enable': [ - 'CONSOLE_FRAMEBUFFER' - ], - 'disable': [ - # Disables would go here. - ]}} - -rootdir = '/opt/builds' -logdir = os.path.join(rootdir, 'logs', 'ipxe') -patchdir = os.path.join(rootdir, 'patches', 'ipxe') -buildroot = os.path.join(rootdir, 'ipxe_build') -srcdir = os.path.join(buildroot, 'src') -configdir = os.path.join(rootdir, 'configs', 'ipxe') -destdir = os.path.join(rootdir, 'built', 'ipxe') -repo = git.Repo(buildroot) -master = repo.branches.master -remote = repo.remote('origin') - -rom_suffixes = ('rom', 'mrom') -rom_types = ('rtl8139', '8086100e', '80861209', '10500940', '10222000', '10ec8139', '1af41000', '8086100f', - '808610d3', '15ad07b0') + def __init__(self, opt_xml, *args, **kwargs): + self.xml = opt_xml + self.flags = [i.text for i in self.xml.findall('opt')] -def doMonkeypatch(fname, changeset): - enable_re = None - disable_re = None - fpath = os.path.join(srcdir, fname) - if changeset['enable']: - enable_re = re.compile((r'^\s*(//#define|#undef)\s+' - r'(?P{0})' - r'(?P\s+/\*.*)?\s*$').format('|'.join(changeset['enable']))) - if changeset['disable']: - disable_re = re.compile((r'^(#define|//#undef)\s+' - r'(?P{0})' - r'(?P\s+/\*.*)?\s*$').format('|'.join(changeset['disable']))) - with open(fpath, 'r') as fh: - configstr = fh.read() - configlines = configstr.splitlines() - for idx, line in enumerate(configlines[:]): - if enable_re: - r = enable_re.search(line) - if r: - configlines[idx] = '#define {0}{1}'.format(r.group('optname'), r.group('comment')) - if disable_re: - r = disable_re.search(line) - if r: - configlines[idx] = '#undef {0}{1}'.format(r.group('optname'), r.group('comment')) - with open(fpath, 'w') as fh: - fh.write('\n'.join(configlines)) - fh.write('\n') - return() +class _Enable(_Opt): + switchtype = 'enable' -def main(): - # Cleanup the repo. - repo.head.reset(commit = 'HEAD', working_tree = True) - if repo.active_branch != master: - master.checkout() - repo.head.reset(commit = 'HEAD', working_tree = True) - repo.git.clean('-xdf') - try: - remote.pull() - except BrokenPipeError: + def __init__(self, opt_xml, *args, **kwargs): + super().__init__(opt_xml, *args, **kwargs) + self.re = re.compile((r'^\s*(//#define|#undef)\s+' + r'(?P{0})' + r'(?P\s+/\*.*)?\s*$').format('|'.join(self.flags))) + + +class _Disable(_Opt): + switchtype = 'disable' + + def __init__(self, opt_xml, *args, **kwargs): + super().__init__(opt_xml, *args, **kwargs) + self.re = re.compile((r'^(#define|//#undef)\s+' + r'(?P{0})' + r'(?P\s+/\*.*)?\s*$').format('|'.join(self.flags))) + + +class _MonkeyPatchFile(object): + def __init__(self, builddir, opts_xml): + self.xml = opts_xml + self.fpath = os.path.join(os.path.abspath(os.path.expanduser(builddir)), + self.xml.attrib.get('file')) + self.opts = [] + if not os.path.isfile(self.fpath): + _log.error('File {0} was due to be monkeypatched but is not found.'.format(self.fpath)) + raise FileNotFoundError('File does not exist') + with open(self.fpath, 'r') as fh: + self.buf = io.StringIO(fh.read()) + self.buf.seek(0, 0) + self.lines = self.buf.read().splitlines() + self.buf.seek(0, 0) + self._get_opts() + + def _get_opts(self): + for opt in self.xml.xpath('./enable|./disable'): + if opt.tag == 'enable': + self.opts.append(_Enable(opt)) + else: + self.opts.append(_Disable(opt)) + return (None) + + def patch(self): + for opt in self.opts: + for idx, line in enumerate(self.lines[:]): + opt_re = opt.re.search(line) + if opt_re: + if opt.switchtype == 'enable': + self.lines[idx] = '#define {0}{1}'.format(opt_re.group('optname'), + opt_re.group('comment')) + else: + self.lines[idx] = '#undef {0}{1}'.format(opt_re.group('optname'), + opt_re.group('comment')) + shutil.copy2(self.fpath, '{0}.orig'.format(self.fpath)) + with open(self.fpath, 'w') as fh: + fh.write('\n'.join(self.lines)) + fh.write('\n') + return(None) + + +class MonkeyPatch(object): + def __init__(self, builddir, switchopts_xml): + self.root = os.path.join(os.path.abspath(os.path.expanduser(builddir)), + os.path.abspath(os.path.expanduser(switchopts_xml.attrib.get('subDir', '.')))) + self.xml = switchopts_xml + self.files = [] + self._get_files() + + def _get_files(self): + for optfile in self.xml.findall('opts'): + self.files.append(_MonkeyPatchFile(self.root, optfile)) + return(None) + + +class _PatchFile(object): + def __init__(self, patchdir, builddir, patchfile_xml): + self.xml = patchfile_xml + self.builddir = os.path.abspath(os.path.expanduser(builddir)) + self.fpath = os.path.join(os.path.abspath(os.path.expanduser(patchdir)), + self.xml.text) + self.strip_level = int(self.xml.attrib.get('stripLevel', 1)) + if not os.path.isfile(self.fpath): + _log.error('Patch file {0} does not exist'.format(self.fpath)) + with open(self.fpath, 'r') as fh: + self.patch_raw = fh.read() + self.patch_obj = patch_ng.PatchSet(self.patch_raw) + + def patch(self): + self.patch_obj.apply(strip = self.strip_level, root = self.builddir) + return(None) + + +class Patch(object): + def __init__(self, builddir, patch_xml): + self.root = os.path.abspath(os.path.expanduser(builddir)) + self.xml = patch_xml + _patch_dir = pathlib.Path(self.xml.attrib.get('patchDir', '.')) + if not _patch_dir.is_absolute(): + self.patch_dir = os.path.join(self.root, str(_patch_dir)) + else: + self.patch_dir = str(_patch_dir) + self.files = [] + + def _get_patches(self): + for patch in self.xml.findall('patchFile'): + self.files.append(_PatchFile(self.patch_dir, self.root, patch)) + return(None) + + +class Builder(object): + def __init__(self, cfg = None): + if not cfg: + cfg = constants.IPXE_DEFAULT_CFG + self.cfg_file = os.path.abspath(os.path.expanduser(cfg)) + self.cfg = config.Config(cfg) + self.xml = self.cfg.xml + self.dest = None + self.builddir = None + self.upstream = None + self.compiler = None + self.monkeypatches = [] + self.patches = [] + self._get_info() + self._get_patch() + self._get_compile() + + def _get_compile(self): + build_xml = self.xml.find('build') + self.compiler = compile.Compiler(self.builddir, + self.dest, + self.upstream, + self.patches, + self.monkeypatches, + build_xml) + + def _get_info(self): + source_xml = self.xml.find('source') + build_xml = self.xml.find('build') + self.dest = os.path.abspath(os.path.expanduser(build_xml.find('dest').text)) + self.upstream = upstream.upstream_parser(source_xml.find('upstream')) + self.builddir = self.upstream.dest + return(None) + + def _get_patch(self): + for patchset in self.xml.xpath('//patchSet'): + for optswitch in patchset.findall('switchOpts'): + self.monkeypatches.append(MonkeyPatch(self.builddir, optswitch)) + for patch in self.xml.xpath('//patch'): + self.patches.append(Patch(self.builddir, patch)) + return(None) + + def build(self): pass - # Patch - for p in patches: - with open(os.path.join(patchdir, p), 'rb') as fh: - patchset = patch_ng.PatchSet(fh) - patchset.apply(strip = 1, root = buildroot) - # "Monkeypatch" - sed-like. - for f, changeset in monkeypatch.items(): - doMonkeypatch(f, changeset) - # Build. Finally! - # TODO: ARM support! - # for d in ('default', 'efi', 'iso', 'legacy', 'roms', 'arm'): - for d in ('default', 'efi', 'iso', 'legacy', 'roms'): - dpath = os.path.join(destdir, d) - os.makedirs(dpath, exist_ok = True) - os.makedirs(logdir, exist_ok = True) - os.chdir(srcdir) - ## Base files - # TODO: ARM support! - # TODO: efi-sb support (secureboot)! - # http://ipxe.org/appnote/buildtargets - # http://ipxe.org/appnote/buildtargets#special_targets - ## BOOTSTRAP ## - with open(os.path.join(logdir, 'all_bootstrap.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'all_bootstrap.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'all', - 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], - stdout = stdout, - stderr = stderr) - for fsrc, fdest in (('undionly.kpxe', 'legacy/bootstrap_{f}'), - ('ipxe.iso', 'iso/bootstrap_bios.iso'), - ('ipxe.usb', 'iso/bootstrap_pxe_usb.img'), - ('ipxe.dsk', 'iso/bootstrap_pxe_floppy.img')): - srcpath = os.path.join(srcdir, 'bin', fsrc) - fname = os.path.basename(srcpath) - destpath = os.path.join(destdir, fdest.format(f = fname)) - shutil.copy2(srcpath, destpath) - for rom in rom_types: - for s in rom_suffixes: - fname = '{0}.{1}'.format(rom, s) - fpath = os.path.join(srcdir, 'bin', fname) - if os.path.isfile(fpath): - shutil.copy2(fpath, - os.path.join(destdir, 'roms', 'bootstrap_{0}'.format(fname))) - # http://ipxe.org/howto/romburning - # https://libvirt.org/formatdomain.html#elementsNICSROM - if rom == '1af41000': - os.symlink(fpath, - os.path.join(destdir, 'roms', 'bootstrap_virtio.rom')) - ## EMBEDDED ## - with open(os.path.join(logdir, 'all.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'all.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'all', - 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], - stdout = stdout, - stderr = stderr) - for fsrc, fdest in (('undionly.kpxe', 'legacy/{f}'), - ('ipxe.iso', 'iso/bios.iso'), - ('ipxe.usb', 'iso/pxe_usb.img'), - ('ipxe.dsk', 'iso/pxe_floppy.img')): - srcpath = os.path.join(srcdir, 'bin', fsrc) - fname = os.path.basename(srcpath) - destpath = os.path.join(destdir, fdest.format(f = fname)) - shutil.copy2(srcpath, destpath) - for rom in rom_types: - for s in rom_suffixes: - fname = '{0}.{1}'.format(rom, s) - fpath = os.path.join(srcdir, 'bin', fname) - if os.path.isfile(fpath): - shutil.copy2(fpath, - os.path.join(destdir, 'roms', fname)) - if rom == '1af41000': - os.symlink(fpath, - os.path.join(destdir, 'roms', 'virtio.rom')) - # DOS/MBR sector/HDD img. - ## BOOTSTRAP ## - with open(os.path.join(logdir, 'bootstrap_hdd.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'bootstrap_hdd.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin/ipxe.hd', - 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], - stdout = stdout, - stderr = stderr) - shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'), - os.path.join(destdir, 'iso', 'bootstrap_legacy_mbr.img')) - ## EMBEDDED ## - with open(os.path.join(logdir, 'hdd.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'hdd.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin/ipxe.hd', - 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], - stdout = stdout, - stderr = stderr) - shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'), - os.path.join(destdir, 'iso', 'legacy_mbr.img')) - # PXE loaders - ## BOOTSTRAP ## - with open(os.path.join(logdir, 'bootstrap_loader.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'bootstrap_loader.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin/ipxe.pxe', - 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], - stdout = stdout, - stderr = stderr) - os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'), - os.path.join(destdir, 'default', 'bootstrap_loader.pxe')) - ## EMBEDDED ## - with open(os.path.join(logdir, 'loader.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'loader.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin/ipxe.pxe', - 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], - stdout = stdout, - stderr = stderr) - os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'), - os.path.join(destdir, 'default', 'loader.pxe')) - # EFI binaries and ISO images - # These have to be done grouped because the eiso stuff APPARENTLY doesn't parse EMBED or lack thereof correctly? - ## BOOTSTRAP ## - ### EFI ### - with open(os.path.join(logdir, 'bootstrap_efi.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'bootstrap_efi.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin-i386-efi/ipxe.efi', - 'bin-x86_64-efi/ipxe.efi', - 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], - stdout = stdout, - stderr = stderr) - shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'), - os.path.join(destdir, 'efi', 'bootstrap_32.efi')) - shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'), - os.path.join(destdir, 'efi', 'bootstrap_64.efi')) - ### UEFI ISO ### - with open(os.path.join(logdir, 'bootstrap_iso.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'bootstrap_iso.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin/ipxe.liso', - 'bin/ipxe.eiso', - 'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))], - stdout = stdout, - stderr = stderr) - os.rename(os.path.join(srcdir, 'bin/ipxe.liso'), - os.path.join(destdir, 'iso', 'bootstrap_legacy.iso')) - os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'), - os.path.join(destdir, 'iso', 'bootstrap_uefi.iso')) - ## EMBEDDED ## - ### EFI ### - with open(os.path.join(logdir, 'efi.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'efi.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin-i386-efi/ipxe.efi', - 'bin-x86_64-efi/ipxe.efi', - 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], - stdout = stdout, - stderr = stderr) - shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'), - os.path.join(destdir, 'efi', '32.efi')) - shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'), - os.path.join(destdir, 'efi', '64.efi')) - ### UEFI ISO ### - with open(os.path.join(logdir, 'iso.stderr'), 'wb') as stderr, \ - open(os.path.join(logdir, 'iso.stdout'), 'wb') as stdout: - subprocess.run(['make', - 'bin/ipxe.liso', - 'bin/ipxe.eiso', - 'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))], - stdout = stdout, - stderr = stderr) - os.rename(os.path.join(srcdir, 'bin/ipxe.liso'), - os.path.join(destdir, 'iso', 'legacy.iso')) - os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'), - os.path.join(destdir, 'iso', 'uefi.iso')) - return() - -if __name__ == '__main__': - main() diff --git a/builder/logger.py b/builder/logger.py new file mode 100644 index 0000000..b5397e2 --- /dev/null +++ b/builder/logger.py @@ -0,0 +1,56 @@ +import logging +import logging.handlers +import os +try: + # https://www.freedesktop.org/software/systemd/python-systemd/journal.html#journalhandler-class + from systemd import journal + _has_journald = True +except ImportError: + _has_journald = False +## +from . import constants + + +def preplog(logfile = None): + if not logfile: + logfile = constants.IPXE_DEFAULT_LOGFILE + # Prep the log file. + logfile = os.path.abspath(os.path.expanduser(logfile)) + os.makedirs(os.path.dirname(logfile), exist_ok = True, mode = 0o0700) + if not os.path.isfile(logfile): + with open(logfile, 'w') as fh: + fh.write('') + os.chmod(logfile, 0o0600) + return(logfile) + + +# And set up logging. +_cfg_args = {'handlers': [], + 'level': logging.DEBUG} +if _has_journald: + # There were some weird changes somewhere along the line. + try: + # But it's *probably* this one. + h = journal.JournalHandler() + except AttributeError: + h = journal.JournaldLogHandler() + # Systemd includes times, so we don't need to. + h.setFormatter(logging.Formatter(style = '{', + fmt = ('{name}:{levelname}:{filename}:' + '{funcName}:{lineno}: {message}'))) + _cfg_args['handlers'].append(h) + +filehandler = logging.handlers.RotatingFileHandler(preplog(), + encoding = 'utf8', + # Disable rotating for now. + # maxBytes = 50000000000, + # backupCount = 30 + ) +filehandler.setFormatter(logging.Formatter(style = '{', + fmt = ('{asctime}:' + '{levelname}:{name}:{filename}:' + '{funcName}:{lineno}: {message}'))) +_cfg_args['handlers'].append(filehandler) +logging.basicConfig(**_cfg_args) +logger = logging.getLogger('iPXE Builder') +logger.info('Logging initialized.') diff --git a/builder/upstream.py b/builder/upstream.py new file mode 100644 index 0000000..7c92744 --- /dev/null +++ b/builder/upstream.py @@ -0,0 +1,134 @@ +import io +import os +import re +import shutil +import tarfile +import zipfile +from urllib.parse import urlparse +## +import requests +try: + import git + _has_git = True +except ImportError: + _has_git = False +## +from . import constants + + +class _Upstream(object): + def __init__(self, upstream_xml, *args, **kwargs): + self.xml = upstream_xml + self.uri = None + self.proto = None + self.host = None + self.port = None + self.path = None + self.parsed_uri = None + self.dest = os.path.abspath(os.path.expanduser(self.xml.find('dest').text)) + os.makedirs(os.path.dirname(self.dest), exist_ok = True, mode = 0o0750) + + def parse_uri(self): + self.parsed_uri = urlparse(self.uri) + self.proto = self.parsed_uri.scheme.lower() + self.host = self.parsed_uri.hostname.lower() + self.port = int(getattr(self.parsed_uri, 'port', constants.DEF_PORTS[self.proto])) + self.path = self.parsed_uri.path + return(None) + + +class Git(_Upstream): + upstreamtype = 'git' + + def __init__(self, upstream_xml, refresh = False, *args, **kwargs): + if not _has_git: + raise RuntimeError('The git module (GitPython) is not installed') + super().__init__(upstream_xml, *args, **kwargs) + self.repo = None + self.remote = None + self.ref = None + self.refresh = refresh + git_xml = self.xml.find('git') + try: + self.ref_name = git_xml.find('ref').text + except AttributeError: + self.ref_name = 'master' + try: + self.ref_type = git_xml.find('refType').text + except AttributeError: + self.ref_type = 'branch' + self.uri = git_xml.find('uri').text + self.parse_uri() + + def fetch(self): + if os.path.isdir(self.dest) and not self.refresh: + self.repo = git.Repo(self.dest) + elif os.path.isdir(self.dest): + shutil.rmtree(self.dest) + if not self.repo: + self.repo = git.Repo.clone_from(self.uri, self.dest) + for r in self.repo.remotes: + if self.uri in r.urls: + self.remote = r + break + if not self.remote: + self.remote = self.repo.create_remote('bootbox_ipxe_upstream', self.uri) + self.remote.pull() + if self.ref_type in ('branch', 'head'): + try: + self.ref = self.repo.branches[self.ref_name] + except IndexError: + self.ref = self.repo.create_head(self.ref_name, self.remote.refs[self.ref_name]) + self.ref.set_tracking_branch(self.ref_name) + self.ref.checkout() + self.repo.head.reset(self.ref, working_tree = True) + elif self.ref_type in ('tag', 'commit', 'rev'): + if self.ref_type == 'tag': + self.ref = self.repo.tags[self.ref_name] + else: + self.ref = self.repo.commit(rev = self.ref_name) + self.repo.head.reset(self.ref, working_tree = True) + self.repo.git.clean('-xdf') + self.remote.pull() + return(None) + + +class Archive(_Upstream): + upstreamtype = 'archive' + + def __init__(self, upstream_xml, *args, **kwargs): + super().__init__(upstream_xml, *args, **kwargs) + self.uri = self.xml.find('archive').text + self.parse_uri() + self.suffix = re.sub(r'^.*\.(?P(tar(\.((g|x)z|bz2?))?|t((g|x)z|bz2?)|zip))$', + r'\g', + os.path.basename(self.path)) + self.req = None + try: + shutil.rmtree(self.dest) + except FileNotFoundError: + pass + os.makedirs(self.dest, mode = 0o0750, exist_ok = True) + + def fetch(self): + self.req = requests.get(self.uri) + buf = io.BytesIO(self.req.content) + if not self.req.ok: + raise RuntimeError('There was an error fetching the source') + if self.suffix == 'zip': + handler = zipfile.ZipFile(buf, 'r') + else: + handler = tarfile.open(fileobj = buf, mode = 'r') + handler.extractall(self.dest) + handler.close() + return(None) + + +def upstream_parser(upstream_xml): + arc = upstream_xml.find('archive') + gitrepo = upstream_xml.find('git') + if arc: + return(Archive(upstream_xml)) + elif gitrepo: + return(Git(upstream_xml)) + return(None)