Automated commit (/opt/dev/infra/gitclass.py)
This commit is contained in:
parent
3dc366595c
commit
6135fb850d
@ -1 +1,4 @@
|
||||
from . import logger
|
||||
from . import config
|
||||
from . import compile
|
||||
from . import ipxe
|
||||
|
||||
69
builder/compile.py
Normal file
69
builder/compile.py
Normal file
@ -0,0 +1,69 @@
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
##
|
||||
from . import constants
|
||||
|
||||
|
||||
_log = logging.getLogger()
|
||||
|
||||
|
||||
class Target(object):
|
||||
def __init__(self, target_xml):
|
||||
self.xml = target_xml
|
||||
self.subdir = self.xml.attrib.get('subDir', '.')
|
||||
self.base = self.xml.attrib.get('baseName')
|
||||
self.target = self.xml.text
|
||||
if not self.base:
|
||||
self.base = os.path.basename(self.target)
|
||||
|
||||
|
||||
class IpxeScript(object):
|
||||
def __init__(self, script_dir, script_xml):
|
||||
self.xml = script_xml
|
||||
self.fpath = os.path.join(os.path.abspath(os.path.expanduser(script_dir)),
|
||||
self.xml.text)
|
||||
self.prefix = self.xml.attrib['prefix']
|
||||
|
||||
|
||||
class Compiler(object):
|
||||
def __init__(self, builddir, destdir, upstream, patches, monkey_patches, build_xml):
|
||||
self.xml = build_xml
|
||||
self.build = os.path.abspath(os.path.expanduser(builddir))
|
||||
self.dest = os.path.abspath(os.path.expanduser(destdir))
|
||||
self.src = upstream.dest
|
||||
self.upstream = upstream
|
||||
self.patches = patches
|
||||
self.monkey_patches = monkey_patches
|
||||
self.targets = []
|
||||
self.scripts = []
|
||||
self._add_targets()
|
||||
|
||||
def _add_targets(self):
|
||||
roms = self.xml.findall('rom')
|
||||
if roms is None:
|
||||
|
||||
|
||||
def make(self):
|
||||
# NOTE: 1af41000 is the firmware used by virtIO
|
||||
self.prep()
|
||||
self.patch()
|
||||
|
||||
def patch(self):
|
||||
for m in self.monkey_patches:
|
||||
for pf in m.files:
|
||||
pf.patch()
|
||||
for p in self.patches:
|
||||
for pf in p.files:
|
||||
pf.patch()
|
||||
return()
|
||||
|
||||
def prep(self):
|
||||
if self.src != self.build:
|
||||
shutil.copytree(self.src, self.build, dirs_exist_ok = True)
|
||||
os.makedirs(self.dest)
|
||||
# These are standard.
|
||||
for d in constants.IPXE_CATEGORIES:
|
||||
dpath = os.path.join(self.dest, d)
|
||||
os.makedirs(dpath, exist_ok = True)
|
||||
173
builder/config.py
Normal file
173
builder/config.py
Normal file
@ -0,0 +1,173 @@
|
||||
import copy
|
||||
import os
|
||||
import logging
|
||||
import re
|
||||
##
|
||||
import requests
|
||||
import requests.auth
|
||||
from lxml import etree
|
||||
##
|
||||
from . import constants
|
||||
|
||||
|
||||
_logger = logging.getLogger()
|
||||
|
||||
|
||||
def create_default_cfg():
|
||||
# Create a stripped sample config.
|
||||
ws_re = re.compile(r'^\s*$')
|
||||
samplexml = constants.IPXE_SAMPLE_CONFIG
|
||||
with open(samplexml, 'rb') as fh:
|
||||
xml = etree.fromstring(fh.read())
|
||||
# Create a stripped sample config.
|
||||
# First we strip comments (and fix the ensuing whitespace).
|
||||
# etree has a .canonicalize(), but it chokes on a default namespace.
|
||||
# https://bugs.launchpad.net/lxml/+bug/1869455
|
||||
# So everything we do is kind of a hack.
|
||||
# for c in xml.xpath("//comment()"):
|
||||
# parent = c.getparent()
|
||||
# parent.remove(c)
|
||||
xmlstr = etree.tostring(xml, with_comments = False, method = 'c14n', pretty_print = True).decode('utf-8')
|
||||
newstr = []
|
||||
for line in xmlstr.splitlines():
|
||||
r = ws_re.search(line)
|
||||
if not r:
|
||||
newstr.append(line.strip())
|
||||
xml = etree.fromstring(''.join(newstr).encode('utf-8'))
|
||||
# Remove text and attr text.
|
||||
xpathq = "descendant-or-self::*[namespace-uri()!='']"
|
||||
for e in xml.xpath(xpathq):
|
||||
if e.tag == '{{{0}}}ipxe'.format(xml.nsmap[None]):
|
||||
continue
|
||||
if e.text is not None and e.text.strip() != '':
|
||||
e.text = ''
|
||||
for k, v in e.attrib.items():
|
||||
if v is not None:
|
||||
e.attrib[k] = ''
|
||||
# Remove multiple children of same type to simplify.
|
||||
for e in xml.xpath(xpathq):
|
||||
if e.tag == '{{{0}}}ipxe'.format(xml.nsmap[None]):
|
||||
continue
|
||||
parent = e.getparent()
|
||||
try:
|
||||
for idx, child in enumerate(parent.findall(e.tag)):
|
||||
if idx == 0:
|
||||
continue
|
||||
parent.remove(child)
|
||||
except AttributeError:
|
||||
pass
|
||||
# And add a comment pointing them to the fully commented config.
|
||||
xml.insert(0, etree.Comment(('\n Please reference the fully commented example.config.xml found either '
|
||||
'at:\n '
|
||||
' * {0}\n * https://git.square-r00t.net/BootBox/tree/builder/'
|
||||
'example.config.xml\n and then configure this according to those '
|
||||
'instructions.\n ').format(samplexml)))
|
||||
return(etree.tostring(xml,
|
||||
pretty_print = True,
|
||||
with_comments = True,
|
||||
with_tail = True,
|
||||
encoding = 'UTF-8',
|
||||
xml_declaration = True))
|
||||
|
||||
|
||||
class Config(object):
|
||||
default_xsd = 'http://schema.xml.r00t2.io/projects/ipxe/build.xsd'
|
||||
default_xml_path = constants.IPXE_DEFAULT_CFG
|
||||
|
||||
def __init__(self, xml_path, *args, **kwargs):
|
||||
if not xml_path:
|
||||
xml_path = self.default_xml_path
|
||||
self.xml_path = os.path.abspath(os.path.expanduser(xml_path))
|
||||
if not os.path.isfile(self.xml_path):
|
||||
with open(self.xml_path, 'wb') as fh:
|
||||
fh.write(create_default_cfg())
|
||||
_logger.error(('{0} does not exist so a sample configuration file has been created in its place. '
|
||||
'Be sure to configure it appropriately.').format(self.default_xml_path))
|
||||
raise ValueError('Config does not exist')
|
||||
else:
|
||||
with open(self.xml_path, 'rb') as fh:
|
||||
self.raw = fh.read()
|
||||
self.xml = None
|
||||
self.xsd = None
|
||||
self.ns_xml = None
|
||||
self.tree = None
|
||||
self.ns_tree = None
|
||||
self.defaults_parser = None
|
||||
self.parse_xml()
|
||||
_logger.info('Instantiated {0}.'.format(type(self).__name__))
|
||||
|
||||
def get_xsd(self):
|
||||
raw_xsd = None
|
||||
base_url = None
|
||||
xsi = self.xml.nsmap.get('xsi', 'http://www.w3.org/2001/XMLSchema-instance')
|
||||
schemaLocation = '{{{0}}}schemaLocation'.format(xsi)
|
||||
schemaURL = self.xml.attrib.get(schemaLocation, self.default_xsd)
|
||||
split_url = schemaURL.split()
|
||||
if len(split_url) == 2: # a properly defined schemaLocation
|
||||
schemaURL = split_url[1]
|
||||
else:
|
||||
schemaURL = split_url[0] # a LAZY schemaLocation
|
||||
if schemaURL.startswith('file://'):
|
||||
schemaURL = re.sub(r'^file://', r'', schemaURL)
|
||||
with open(schemaURL, 'rb') as fh:
|
||||
raw_xsd = fh.read()
|
||||
base_url = os.path.dirname(schemaURL) + '/'
|
||||
else:
|
||||
req = requests.get(schemaURL)
|
||||
if not req.ok:
|
||||
raise RuntimeError('Could not download XSD')
|
||||
raw_xsd = req.content
|
||||
base_url = os.path.split(req.url)[0] + '/' # This makes me feel dirty.
|
||||
self.xsd = etree.XMLSchema(etree.XML(raw_xsd, base_url = base_url))
|
||||
return(None)
|
||||
|
||||
def parse_xml(self):
|
||||
self.parse_raw()
|
||||
self.get_xsd()
|
||||
self.populate_defaults()
|
||||
self.validate()
|
||||
return(None)
|
||||
|
||||
def parse_raw(self, parser = None):
|
||||
self.xml = etree.fromstring(self.raw, parser = parser)
|
||||
self.ns_xml = etree.fromstring(self.raw, parser = parser)
|
||||
self.tree = self.xml.getroottree()
|
||||
self.ns_tree = self.ns_xml.getroottree()
|
||||
self.tree.xinclude()
|
||||
self.ns_tree.xinclude()
|
||||
self.strip_ns()
|
||||
return(None)
|
||||
|
||||
def populate_defaults(self):
|
||||
if not self.xsd:
|
||||
self.get_xsd()
|
||||
if not self.defaults_parser:
|
||||
self.defaults_parser = etree.XMLParser(schema = self.xsd, attribute_defaults = True)
|
||||
self.parse_raw(parser = self.defaults_parser)
|
||||
return(None)
|
||||
|
||||
def remove_defaults(self):
|
||||
self.parse_raw()
|
||||
return(None)
|
||||
|
||||
def strip_ns(self, obj = None):
|
||||
# https://stackoverflow.com/questions/30232031/how-can-i-strip-namespaces-out-of-an-lxml-tree/30233635#30233635
|
||||
xpathq = "descendant-or-self::*[namespace-uri()!='']"
|
||||
if not obj:
|
||||
for x in (self.tree, self.xml):
|
||||
for e in x.xpath(xpathq):
|
||||
e.tag = etree.QName(e).localname
|
||||
elif isinstance(obj, (etree._Element, etree._ElementTree)):
|
||||
obj = copy.deepcopy(obj)
|
||||
for e in obj.xpath(xpathq):
|
||||
e.tag = etree.QName(e).localname
|
||||
return(obj)
|
||||
else:
|
||||
raise ValueError('Did not know how to parse obj parameter')
|
||||
return(None)
|
||||
|
||||
def validate(self):
|
||||
if not self.xsd:
|
||||
self.get_xsd()
|
||||
self.xsd.assertValid(self.ns_tree)
|
||||
return(None)
|
||||
29
builder/constants.py
Normal file
29
builder/constants.py
Normal file
@ -0,0 +1,29 @@
|
||||
import os
|
||||
|
||||
IPXE_GIT_URI = 'git://git.ipxe.org/ipxe.git'
|
||||
IPXE_ROM_SUFFIXES = ('rom', 'mrom')
|
||||
IPXE_ROM_FIRMWARES = ('rtl8139',
|
||||
'8086100e',
|
||||
'80861209',
|
||||
'10500940',
|
||||
'10222000',
|
||||
'10ec8139',
|
||||
'1af41000', # firmware used by virtIO
|
||||
'8086100f',
|
||||
'808610d3',
|
||||
'15ad07b0')
|
||||
_cur_dir = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
|
||||
IPXE_SAMPLE_CONFIG = os.path.abspath(os.path.join(_cur_dir, 'example.config.xml'))
|
||||
IPXE_DEFAULT_CFG = '~/.config/bootbox/ipxe_build.xml'
|
||||
if os.geteuid() == 0:
|
||||
IPXE_DEFAULT_LOGFILE = '/var/log/bootbox/ipxe.log'
|
||||
else:
|
||||
IPXE_DEFAULT_LOGFILE = '~/.cache/bootbox/ipxe.log'
|
||||
DEF_PORTS = {'git': 9418,
|
||||
'http': 80,
|
||||
'https': 443}
|
||||
IPXE_CATEGORIES = ('default',
|
||||
'efi',
|
||||
'iso',
|
||||
'legacy',
|
||||
'roms')
|
||||
@ -2,9 +2,26 @@
|
||||
<ipxe xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="https://router.r00t2.io/boootbox/ipxe/build/"
|
||||
xsi:schemaLocation="https://router.r00t2.io/boootbox/ipxe/build/ http://schema.xml.r00t2.io/projects/ipxe/build.xsd">
|
||||
<!--
|
||||
Define where the source code should go, where it comes from, and any modifications if desired.
|
||||
-->
|
||||
<source srcDir="/opt/builds/ipxe_build">
|
||||
<upstream>
|
||||
<git>git://git.ipxe.org/ipxe.git</git>
|
||||
<!--
|
||||
Upstream supports two types, "git" (shown below) and "archive". If you use archive, it must be an HTTP file, an
|
||||
HTTPS file, or a path on the local filesystem (file:///path/to/file.tar.xz). archive supports:
|
||||
* .tar.bz2/.tbz/.tbz2
|
||||
* .tar.xz/.txz
|
||||
* .tar.gz/.tgz
|
||||
* .tar
|
||||
* .zip
|
||||
If you use "git", you must have python-git (https://pypi.org/project/GitPython/) installed.
|
||||
-->
|
||||
<git>
|
||||
<refType>branch</refType>
|
||||
<ref>master</ref>
|
||||
<uri>git://git.ipxe.org/ipxe.git</uri>
|
||||
</git>
|
||||
<dest>/opt/builds/ipxe_src</dest>
|
||||
</upstream>
|
||||
<patchSet>
|
||||
@ -75,10 +92,57 @@
|
||||
</opts>
|
||||
</switchOpts>
|
||||
<patch patchDir="/opt/builds/patches/ipxe">
|
||||
<patchFile>git-version.patch</patchFile>
|
||||
<patchFile>banner.patch</patchFile>
|
||||
<patchFile>efi-iso.patch</patchFile>
|
||||
<patchFile stripLevel="1">git-version.patch</patchFile>
|
||||
<patchFile stripLevel="1">banner.patch</patchFile>
|
||||
<patchFile stripLevel="1">efi-iso.patch</patchFile>
|
||||
</patch>
|
||||
</patchSet>
|
||||
</source>
|
||||
<!--
|
||||
The other attributes enable additional target images. "pxe" is a PXE image, "undi" is an UNDI (RFC4578§2.2)
|
||||
kernel/NBP, both of which will allow you to chainload iPXE from vanilla PXE.
|
||||
-->
|
||||
<build
|
||||
iso="true"
|
||||
usb="true"
|
||||
floppy="true"
|
||||
mbr="true"
|
||||
pxe="true"
|
||||
undi="true">
|
||||
<!--
|
||||
If scripts is specified, multiple firmware sets/images will be built along with a non-embedded version.
|
||||
The required "prefix" attribute is the file prefix that will be appended.
|
||||
See: https://ipxe.org/scripting
|
||||
-->
|
||||
<scripts srcDir="/opt/builds/config/ipxe">
|
||||
<script prefix="bootstrap">bootstrap.ipxe</script>
|
||||
<script prefix="">chain-default.ipxe</script>
|
||||
</scripts>
|
||||
<!--
|
||||
The directory where built firmware/images go. They will have respective subdirectories to maintain a clean
|
||||
hierarchy.
|
||||
-->
|
||||
<dest>/opt/builds/built/ipxe</dest>
|
||||
<!--
|
||||
The default is all firmware ("make everything").
|
||||
Note that 1af41000 is the firmware used by VirtIO, and will be symlinked as such if built.
|
||||
-->
|
||||
<!--
|
||||
<roms>
|
||||
<rom>rtl8139</rom>
|
||||
<rom>8086100e</rom>
|
||||
<rom>80861209</rom>
|
||||
</roms>
|
||||
-->
|
||||
<!--
|
||||
This is optional. If you introduce patches that define additional images that don't follow the above pattern
|
||||
(such as eworm's EFI ISO patch), you can define them here.
|
||||
-->
|
||||
<extraTargets>
|
||||
<target subDir="efi" baseName="32.efi">bin-i386-efi/ipxe.efi</target>
|
||||
<target subDir="efi" baseName="64.efi">bin-x86_64-efi/ipxe.efi</target>
|
||||
<target subDir="iso" baseName="legacy.iso">bin/ipxe.liso</target>
|
||||
<target subDir="iso" baseName="uefi.iso">bin/ipxe.eiso</target>
|
||||
</extraTargets>
|
||||
</build>
|
||||
</ipxe>
|
||||
|
||||
297
builder/ipxe.OLD.py
Normal file
297
builder/ipxe.OLD.py
Normal file
@ -0,0 +1,297 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
##
|
||||
import git # https://pypi.org/project/GitPython/
|
||||
import patch_ng # https://pypi.org/project/patch-ng/
|
||||
|
||||
|
||||
patches = ['git-version.patch', 'banner.patch', 'efi-iso.patch']
|
||||
|
||||
monkeypatch = {
|
||||
'config/general.h': {
|
||||
'enable': [
|
||||
'NET_PROTO_IPV6',
|
||||
'DOWNLOAD_PROTO_HTTPS',
|
||||
'DOWNLOAD_PROTO_FTP',
|
||||
'DOWNLOAD_PROTO_NFS',
|
||||
'HTTP_AUTH_NTLM',
|
||||
# 'HTTP_ENC_PEERDIST',
|
||||
# 'HTTP_HACK_GCE',
|
||||
'NSLOOKUP_CMD',
|
||||
'TIME_CMD',
|
||||
'DIGEST_CMD',
|
||||
'LOTEST_CMD',
|
||||
'VLAN_CMD',
|
||||
# 'PXE_CMD', # Causes EFI to fail.
|
||||
'REBOOT_CMD',
|
||||
'POWEROFF_CMD',
|
||||
# 'IMAGE_NBI', # I *think* this causes EFI to fail. Can I build it directly as a target w/o enabling?
|
||||
# 'IMAGE_ELF', # Causes EFI to fail, and is auto-enabled for what needs it.
|
||||
# 'IMAGE_MULTIBOOT', # Also enabled by default for MBR builds
|
||||
# 'IMAGE_SCRIPT', # Enabled where needed etc.
|
||||
# 'IMAGE_BZIMAGE', # http://lists.ipxe.org/pipermail/ipxe-devel/2017-March/005510.html
|
||||
# 'IMAGE_COMBOOT', # Not really necessary since iPXE has native menus.
|
||||
# 'IMAGE_EFI', # Enabled by default for EFI builds, and causes build error on bin/ipxe.dsk.tmp
|
||||
# 'IMAGE_PXE', # EFI builds fail with this. related to PXE_STACK/PXE_CMD?
|
||||
'IMAGE_TRUST_CMD',
|
||||
'PCI_CMD',
|
||||
'PARAM_CMD',
|
||||
'NEIGHBOUR_CMD',
|
||||
'PING_CMD',
|
||||
'CONSOLE_CMD',
|
||||
'IPSTAT_CMD',
|
||||
'PROFSTAT_CMD',
|
||||
'NTP_CMD',
|
||||
'CERT_CMD'
|
||||
],
|
||||
'disable': [
|
||||
# 'CRYPTO_80211_WEP',
|
||||
# 'CRYPTO_80211_WPA',
|
||||
# 'CRYPTO_80211_WPA2',
|
||||
# 'IWMGMT_CMD'
|
||||
]},
|
||||
'config/console.h': {
|
||||
'enable': [
|
||||
'CONSOLE_FRAMEBUFFER'
|
||||
],
|
||||
'disable': [
|
||||
# Disables would go here.
|
||||
]}}
|
||||
|
||||
rootdir = '/opt/builds'
|
||||
logdir = os.path.join(rootdir, 'logs', 'ipxe')
|
||||
patchdir = os.path.join(rootdir, 'patches', 'ipxe')
|
||||
buildroot = os.path.join(rootdir, 'ipxe_build')
|
||||
srcdir = os.path.join(buildroot, 'src')
|
||||
configdir = os.path.join(rootdir, 'configs', 'ipxe')
|
||||
destdir = os.path.join(rootdir, 'built', 'ipxe')
|
||||
repo = git.Repo(buildroot)
|
||||
master = repo.branches.master
|
||||
remote = repo.remote('origin')
|
||||
|
||||
rom_suffixes = ('rom', 'mrom')
|
||||
rom_types = ('rtl8139', '8086100e', '80861209', '10500940', '10222000', '10ec8139', '1af41000', '8086100f',
|
||||
'808610d3', '15ad07b0')
|
||||
|
||||
|
||||
def doMonkeypatch(fname, changeset):
|
||||
enable_re = None
|
||||
disable_re = None
|
||||
fpath = os.path.join(srcdir, fname)
|
||||
if changeset['enable']:
|
||||
enable_re = re.compile((r'^\s*(//#define|#undef)\s+'
|
||||
r'(?P<optname>{0})'
|
||||
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['enable'])))
|
||||
if changeset['disable']:
|
||||
disable_re = re.compile((r'^(#define|//#undef)\s+'
|
||||
r'(?P<optname>{0})'
|
||||
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['disable'])))
|
||||
with open(fpath, 'r') as fh:
|
||||
configstr = fh.read()
|
||||
configlines = configstr.splitlines()
|
||||
for idx, line in enumerate(configlines[:]):
|
||||
if enable_re:
|
||||
r = enable_re.search(line)
|
||||
if r:
|
||||
configlines[idx] = '#define {0}{1}'.format(r.group('optname'), r.group('comment'))
|
||||
if disable_re:
|
||||
r = disable_re.search(line)
|
||||
if r:
|
||||
configlines[idx] = '#undef {0}{1}'.format(r.group('optname'), r.group('comment'))
|
||||
with open(fpath, 'w') as fh:
|
||||
fh.write('\n'.join(configlines))
|
||||
fh.write('\n')
|
||||
return()
|
||||
|
||||
def main():
|
||||
# Cleanup the repo.
|
||||
repo.head.reset(commit = 'HEAD', working_tree = True)
|
||||
if repo.active_branch != master:
|
||||
master.checkout()
|
||||
repo.head.reset(commit = 'HEAD', working_tree = True)
|
||||
repo.git.clean('-xdf')
|
||||
try:
|
||||
remote.pull()
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
# Patch
|
||||
for p in patches:
|
||||
with open(os.path.join(patchdir, p), 'rb') as fh:
|
||||
patchset = patch_ng.PatchSet(fh)
|
||||
patchset.apply(strip = 1, root = buildroot)
|
||||
# "Monkeypatch" - sed-like.
|
||||
for f, changeset in monkeypatch.items():
|
||||
doMonkeypatch(f, changeset)
|
||||
# Build. Finally!
|
||||
# TODO: ARM support!
|
||||
# for d in ('default', 'efi', 'iso', 'legacy', 'roms', 'arm'):
|
||||
for d in ('default', 'efi', 'iso', 'legacy', 'roms'):
|
||||
dpath = os.path.join(destdir, d)
|
||||
os.makedirs(dpath, exist_ok = True)
|
||||
os.makedirs(logdir, exist_ok = True)
|
||||
os.chdir(srcdir)
|
||||
## Base files
|
||||
# TODO: ARM support!
|
||||
# TODO: efi-sb support (secureboot)!
|
||||
# http://ipxe.org/appnote/buildtargets
|
||||
# http://ipxe.org/appnote/buildtargets#special_targets
|
||||
## BOOTSTRAP ##
|
||||
with open(os.path.join(logdir, 'all_bootstrap.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'all_bootstrap.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'all',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
for fsrc, fdest in (('undionly.kpxe', 'legacy/bootstrap_{f}'),
|
||||
('ipxe.iso', 'iso/bootstrap_bios.iso'),
|
||||
('ipxe.usb', 'iso/bootstrap_pxe_usb.img'),
|
||||
('ipxe.dsk', 'iso/bootstrap_pxe_floppy.img')):
|
||||
srcpath = os.path.join(srcdir, 'bin', fsrc)
|
||||
fname = os.path.basename(srcpath)
|
||||
destpath = os.path.join(destdir, fdest.format(f = fname))
|
||||
shutil.copy2(srcpath, destpath)
|
||||
for rom in rom_types:
|
||||
for s in rom_suffixes:
|
||||
fname = '{0}.{1}'.format(rom, s)
|
||||
fpath = os.path.join(srcdir, 'bin', fname)
|
||||
if os.path.isfile(fpath):
|
||||
shutil.copy2(fpath,
|
||||
os.path.join(destdir, 'roms', 'bootstrap_{0}'.format(fname)))
|
||||
# http://ipxe.org/howto/romburning
|
||||
# https://libvirt.org/formatdomain.html#elementsNICSROM
|
||||
if rom == '1af41000':
|
||||
os.symlink(fpath,
|
||||
os.path.join(destdir, 'roms', 'bootstrap_virtio.rom'))
|
||||
## EMBEDDED ##
|
||||
with open(os.path.join(logdir, 'all.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'all.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'all',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
for fsrc, fdest in (('undionly.kpxe', 'legacy/{f}'),
|
||||
('ipxe.iso', 'iso/bios.iso'),
|
||||
('ipxe.usb', 'iso/pxe_usb.img'),
|
||||
('ipxe.dsk', 'iso/pxe_floppy.img')):
|
||||
srcpath = os.path.join(srcdir, 'bin', fsrc)
|
||||
fname = os.path.basename(srcpath)
|
||||
destpath = os.path.join(destdir, fdest.format(f = fname))
|
||||
shutil.copy2(srcpath, destpath)
|
||||
for rom in rom_types:
|
||||
for s in rom_suffixes:
|
||||
fname = '{0}.{1}'.format(rom, s)
|
||||
fpath = os.path.join(srcdir, 'bin', fname)
|
||||
if os.path.isfile(fpath):
|
||||
shutil.copy2(fpath,
|
||||
os.path.join(destdir, 'roms', fname))
|
||||
if rom == '1af41000':
|
||||
os.symlink(fpath,
|
||||
os.path.join(destdir, 'roms', 'virtio.rom'))
|
||||
# DOS/MBR sector/HDD img.
|
||||
## BOOTSTRAP ##
|
||||
with open(os.path.join(logdir, 'bootstrap_hdd.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_hdd.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.hd',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
|
||||
os.path.join(destdir, 'iso', 'bootstrap_legacy_mbr.img'))
|
||||
## EMBEDDED ##
|
||||
with open(os.path.join(logdir, 'hdd.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'hdd.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.hd',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
|
||||
os.path.join(destdir, 'iso', 'legacy_mbr.img'))
|
||||
# PXE loaders
|
||||
## BOOTSTRAP ##
|
||||
with open(os.path.join(logdir, 'bootstrap_loader.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_loader.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.pxe',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
|
||||
os.path.join(destdir, 'default', 'bootstrap_loader.pxe'))
|
||||
## EMBEDDED ##
|
||||
with open(os.path.join(logdir, 'loader.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'loader.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.pxe',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
|
||||
os.path.join(destdir, 'default', 'loader.pxe'))
|
||||
# EFI binaries and ISO images
|
||||
# These have to be done grouped because the eiso stuff APPARENTLY doesn't parse EMBED or lack thereof correctly?
|
||||
## BOOTSTRAP ##
|
||||
### EFI ###
|
||||
with open(os.path.join(logdir, 'bootstrap_efi.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_efi.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin-i386-efi/ipxe.efi',
|
||||
'bin-x86_64-efi/ipxe.efi',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', 'bootstrap_32.efi'))
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', 'bootstrap_64.efi'))
|
||||
### UEFI ISO ###
|
||||
with open(os.path.join(logdir, 'bootstrap_iso.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_iso.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.liso',
|
||||
'bin/ipxe.eiso',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
|
||||
os.path.join(destdir, 'iso', 'bootstrap_legacy.iso'))
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
|
||||
os.path.join(destdir, 'iso', 'bootstrap_uefi.iso'))
|
||||
## EMBEDDED ##
|
||||
### EFI ###
|
||||
with open(os.path.join(logdir, 'efi.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'efi.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin-i386-efi/ipxe.efi',
|
||||
'bin-x86_64-efi/ipxe.efi',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', '32.efi'))
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', '64.efi'))
|
||||
### UEFI ISO ###
|
||||
with open(os.path.join(logdir, 'iso.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'iso.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.liso',
|
||||
'bin/ipxe.eiso',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
|
||||
os.path.join(destdir, 'iso', 'legacy.iso'))
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
|
||||
os.path.join(destdir, 'iso', 'uefi.iso'))
|
||||
return()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
449
builder/ipxe.py
449
builder/ipxe.py
@ -1,297 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import io
|
||||
import os
|
||||
import logging
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
##
|
||||
import git # https://pypi.org/project/GitPython/
|
||||
import patch_ng # https://pypi.org/project/patch-ng/
|
||||
import patch_ng
|
||||
##
|
||||
from . import config
|
||||
from . import constants
|
||||
from . import upstream
|
||||
from . import compile
|
||||
|
||||
_log = logging.getLogger()
|
||||
|
||||
|
||||
patches = ['git-version.patch', 'banner.patch', 'efi-iso.patch']
|
||||
class _Opt(object):
|
||||
switchtype = None
|
||||
|
||||
monkeypatch = {
|
||||
'config/general.h': {
|
||||
'enable': [
|
||||
'NET_PROTO_IPV6',
|
||||
'DOWNLOAD_PROTO_HTTPS',
|
||||
'DOWNLOAD_PROTO_FTP',
|
||||
'DOWNLOAD_PROTO_NFS',
|
||||
'HTTP_AUTH_NTLM',
|
||||
# 'HTTP_ENC_PEERDIST',
|
||||
# 'HTTP_HACK_GCE',
|
||||
'NSLOOKUP_CMD',
|
||||
'TIME_CMD',
|
||||
'DIGEST_CMD',
|
||||
'LOTEST_CMD',
|
||||
'VLAN_CMD',
|
||||
# 'PXE_CMD', # Causes EFI to fail.
|
||||
'REBOOT_CMD',
|
||||
'POWEROFF_CMD',
|
||||
# 'IMAGE_NBI', # I *think* this causes EFI to fail. Can I build it directly as a target w/o enabling?
|
||||
# 'IMAGE_ELF', # Causes EFI to fail, and is auto-enabled for what needs it.
|
||||
# 'IMAGE_MULTIBOOT', # Also enabled by default for MBR builds
|
||||
# 'IMAGE_SCRIPT', # Enabled where needed etc.
|
||||
# 'IMAGE_BZIMAGE', # http://lists.ipxe.org/pipermail/ipxe-devel/2017-March/005510.html
|
||||
# 'IMAGE_COMBOOT', # Not really necessary since iPXE has native menus.
|
||||
# 'IMAGE_EFI', # Enabled by default for EFI builds, and causes build error on bin/ipxe.dsk.tmp
|
||||
# 'IMAGE_PXE', # EFI builds fail with this. related to PXE_STACK/PXE_CMD?
|
||||
'IMAGE_TRUST_CMD',
|
||||
'PCI_CMD',
|
||||
'PARAM_CMD',
|
||||
'NEIGHBOUR_CMD',
|
||||
'PING_CMD',
|
||||
'CONSOLE_CMD',
|
||||
'IPSTAT_CMD',
|
||||
'PROFSTAT_CMD',
|
||||
'NTP_CMD',
|
||||
'CERT_CMD'
|
||||
],
|
||||
'disable': [
|
||||
# 'CRYPTO_80211_WEP',
|
||||
# 'CRYPTO_80211_WPA',
|
||||
# 'CRYPTO_80211_WPA2',
|
||||
# 'IWMGMT_CMD'
|
||||
]},
|
||||
'config/console.h': {
|
||||
'enable': [
|
||||
'CONSOLE_FRAMEBUFFER'
|
||||
],
|
||||
'disable': [
|
||||
# Disables would go here.
|
||||
]}}
|
||||
|
||||
rootdir = '/opt/builds'
|
||||
logdir = os.path.join(rootdir, 'logs', 'ipxe')
|
||||
patchdir = os.path.join(rootdir, 'patches', 'ipxe')
|
||||
buildroot = os.path.join(rootdir, 'ipxe_build')
|
||||
srcdir = os.path.join(buildroot, 'src')
|
||||
configdir = os.path.join(rootdir, 'configs', 'ipxe')
|
||||
destdir = os.path.join(rootdir, 'built', 'ipxe')
|
||||
repo = git.Repo(buildroot)
|
||||
master = repo.branches.master
|
||||
remote = repo.remote('origin')
|
||||
|
||||
rom_suffixes = ('rom', 'mrom')
|
||||
rom_types = ('rtl8139', '8086100e', '80861209', '10500940', '10222000', '10ec8139', '1af41000', '8086100f',
|
||||
'808610d3', '15ad07b0')
|
||||
def __init__(self, opt_xml, *args, **kwargs):
|
||||
self.xml = opt_xml
|
||||
self.flags = [i.text for i in self.xml.findall('opt')]
|
||||
|
||||
|
||||
def doMonkeypatch(fname, changeset):
|
||||
enable_re = None
|
||||
disable_re = None
|
||||
fpath = os.path.join(srcdir, fname)
|
||||
if changeset['enable']:
|
||||
enable_re = re.compile((r'^\s*(//#define|#undef)\s+'
|
||||
class _Enable(_Opt):
|
||||
switchtype = 'enable'
|
||||
|
||||
def __init__(self, opt_xml, *args, **kwargs):
|
||||
super().__init__(opt_xml, *args, **kwargs)
|
||||
self.re = re.compile((r'^\s*(//#define|#undef)\s+'
|
||||
r'(?P<optname>{0})'
|
||||
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['enable'])))
|
||||
if changeset['disable']:
|
||||
disable_re = re.compile((r'^(#define|//#undef)\s+'
|
||||
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(self.flags)))
|
||||
|
||||
|
||||
class _Disable(_Opt):
|
||||
switchtype = 'disable'
|
||||
|
||||
def __init__(self, opt_xml, *args, **kwargs):
|
||||
super().__init__(opt_xml, *args, **kwargs)
|
||||
self.re = re.compile((r'^(#define|//#undef)\s+'
|
||||
r'(?P<optname>{0})'
|
||||
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(changeset['disable'])))
|
||||
with open(fpath, 'r') as fh:
|
||||
configstr = fh.read()
|
||||
configlines = configstr.splitlines()
|
||||
for idx, line in enumerate(configlines[:]):
|
||||
if enable_re:
|
||||
r = enable_re.search(line)
|
||||
if r:
|
||||
configlines[idx] = '#define {0}{1}'.format(r.group('optname'), r.group('comment'))
|
||||
if disable_re:
|
||||
r = disable_re.search(line)
|
||||
if r:
|
||||
configlines[idx] = '#undef {0}{1}'.format(r.group('optname'), r.group('comment'))
|
||||
with open(fpath, 'w') as fh:
|
||||
fh.write('\n'.join(configlines))
|
||||
r'(?P<comment>\s+/\*.*)?\s*$').format('|'.join(self.flags)))
|
||||
|
||||
|
||||
class _MonkeyPatchFile(object):
|
||||
def __init__(self, builddir, opts_xml):
|
||||
self.xml = opts_xml
|
||||
self.fpath = os.path.join(os.path.abspath(os.path.expanduser(builddir)),
|
||||
self.xml.attrib.get('file'))
|
||||
self.opts = []
|
||||
if not os.path.isfile(self.fpath):
|
||||
_log.error('File {0} was due to be monkeypatched but is not found.'.format(self.fpath))
|
||||
raise FileNotFoundError('File does not exist')
|
||||
with open(self.fpath, 'r') as fh:
|
||||
self.buf = io.StringIO(fh.read())
|
||||
self.buf.seek(0, 0)
|
||||
self.lines = self.buf.read().splitlines()
|
||||
self.buf.seek(0, 0)
|
||||
self._get_opts()
|
||||
|
||||
def _get_opts(self):
|
||||
for opt in self.xml.xpath('./enable|./disable'):
|
||||
if opt.tag == 'enable':
|
||||
self.opts.append(_Enable(opt))
|
||||
else:
|
||||
self.opts.append(_Disable(opt))
|
||||
return (None)
|
||||
|
||||
def patch(self):
|
||||
for opt in self.opts:
|
||||
for idx, line in enumerate(self.lines[:]):
|
||||
opt_re = opt.re.search(line)
|
||||
if opt_re:
|
||||
if opt.switchtype == 'enable':
|
||||
self.lines[idx] = '#define {0}{1}'.format(opt_re.group('optname'),
|
||||
opt_re.group('comment'))
|
||||
else:
|
||||
self.lines[idx] = '#undef {0}{1}'.format(opt_re.group('optname'),
|
||||
opt_re.group('comment'))
|
||||
shutil.copy2(self.fpath, '{0}.orig'.format(self.fpath))
|
||||
with open(self.fpath, 'w') as fh:
|
||||
fh.write('\n'.join(self.lines))
|
||||
fh.write('\n')
|
||||
return()
|
||||
return(None)
|
||||
|
||||
def main():
|
||||
# Cleanup the repo.
|
||||
repo.head.reset(commit = 'HEAD', working_tree = True)
|
||||
if repo.active_branch != master:
|
||||
master.checkout()
|
||||
repo.head.reset(commit = 'HEAD', working_tree = True)
|
||||
repo.git.clean('-xdf')
|
||||
try:
|
||||
remote.pull()
|
||||
except BrokenPipeError:
|
||||
|
||||
class MonkeyPatch(object):
|
||||
def __init__(self, builddir, switchopts_xml):
|
||||
self.root = os.path.join(os.path.abspath(os.path.expanduser(builddir)),
|
||||
os.path.abspath(os.path.expanduser(switchopts_xml.attrib.get('subDir', '.'))))
|
||||
self.xml = switchopts_xml
|
||||
self.files = []
|
||||
self._get_files()
|
||||
|
||||
def _get_files(self):
|
||||
for optfile in self.xml.findall('opts'):
|
||||
self.files.append(_MonkeyPatchFile(self.root, optfile))
|
||||
return(None)
|
||||
|
||||
|
||||
class _PatchFile(object):
|
||||
def __init__(self, patchdir, builddir, patchfile_xml):
|
||||
self.xml = patchfile_xml
|
||||
self.builddir = os.path.abspath(os.path.expanduser(builddir))
|
||||
self.fpath = os.path.join(os.path.abspath(os.path.expanduser(patchdir)),
|
||||
self.xml.text)
|
||||
self.strip_level = int(self.xml.attrib.get('stripLevel', 1))
|
||||
if not os.path.isfile(self.fpath):
|
||||
_log.error('Patch file {0} does not exist'.format(self.fpath))
|
||||
with open(self.fpath, 'r') as fh:
|
||||
self.patch_raw = fh.read()
|
||||
self.patch_obj = patch_ng.PatchSet(self.patch_raw)
|
||||
|
||||
def patch(self):
|
||||
self.patch_obj.apply(strip = self.strip_level, root = self.builddir)
|
||||
return(None)
|
||||
|
||||
|
||||
class Patch(object):
|
||||
def __init__(self, builddir, patch_xml):
|
||||
self.root = os.path.abspath(os.path.expanduser(builddir))
|
||||
self.xml = patch_xml
|
||||
_patch_dir = pathlib.Path(self.xml.attrib.get('patchDir', '.'))
|
||||
if not _patch_dir.is_absolute():
|
||||
self.patch_dir = os.path.join(self.root, str(_patch_dir))
|
||||
else:
|
||||
self.patch_dir = str(_patch_dir)
|
||||
self.files = []
|
||||
|
||||
def _get_patches(self):
|
||||
for patch in self.xml.findall('patchFile'):
|
||||
self.files.append(_PatchFile(self.patch_dir, self.root, patch))
|
||||
return(None)
|
||||
|
||||
|
||||
class Builder(object):
|
||||
def __init__(self, cfg = None):
|
||||
if not cfg:
|
||||
cfg = constants.IPXE_DEFAULT_CFG
|
||||
self.cfg_file = os.path.abspath(os.path.expanduser(cfg))
|
||||
self.cfg = config.Config(cfg)
|
||||
self.xml = self.cfg.xml
|
||||
self.dest = None
|
||||
self.builddir = None
|
||||
self.upstream = None
|
||||
self.compiler = None
|
||||
self.monkeypatches = []
|
||||
self.patches = []
|
||||
self._get_info()
|
||||
self._get_patch()
|
||||
self._get_compile()
|
||||
|
||||
def _get_compile(self):
|
||||
build_xml = self.xml.find('build')
|
||||
self.compiler = compile.Compiler(self.builddir,
|
||||
self.dest,
|
||||
self.upstream,
|
||||
self.patches,
|
||||
self.monkeypatches,
|
||||
build_xml)
|
||||
|
||||
def _get_info(self):
|
||||
source_xml = self.xml.find('source')
|
||||
build_xml = self.xml.find('build')
|
||||
self.dest = os.path.abspath(os.path.expanduser(build_xml.find('dest').text))
|
||||
self.upstream = upstream.upstream_parser(source_xml.find('upstream'))
|
||||
self.builddir = self.upstream.dest
|
||||
return(None)
|
||||
|
||||
def _get_patch(self):
|
||||
for patchset in self.xml.xpath('//patchSet'):
|
||||
for optswitch in patchset.findall('switchOpts'):
|
||||
self.monkeypatches.append(MonkeyPatch(self.builddir, optswitch))
|
||||
for patch in self.xml.xpath('//patch'):
|
||||
self.patches.append(Patch(self.builddir, patch))
|
||||
return(None)
|
||||
|
||||
def build(self):
|
||||
pass
|
||||
# Patch
|
||||
for p in patches:
|
||||
with open(os.path.join(patchdir, p), 'rb') as fh:
|
||||
patchset = patch_ng.PatchSet(fh)
|
||||
patchset.apply(strip = 1, root = buildroot)
|
||||
# "Monkeypatch" - sed-like.
|
||||
for f, changeset in monkeypatch.items():
|
||||
doMonkeypatch(f, changeset)
|
||||
# Build. Finally!
|
||||
# TODO: ARM support!
|
||||
# for d in ('default', 'efi', 'iso', 'legacy', 'roms', 'arm'):
|
||||
for d in ('default', 'efi', 'iso', 'legacy', 'roms'):
|
||||
dpath = os.path.join(destdir, d)
|
||||
os.makedirs(dpath, exist_ok = True)
|
||||
os.makedirs(logdir, exist_ok = True)
|
||||
os.chdir(srcdir)
|
||||
## Base files
|
||||
# TODO: ARM support!
|
||||
# TODO: efi-sb support (secureboot)!
|
||||
# http://ipxe.org/appnote/buildtargets
|
||||
# http://ipxe.org/appnote/buildtargets#special_targets
|
||||
## BOOTSTRAP ##
|
||||
with open(os.path.join(logdir, 'all_bootstrap.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'all_bootstrap.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'all',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
for fsrc, fdest in (('undionly.kpxe', 'legacy/bootstrap_{f}'),
|
||||
('ipxe.iso', 'iso/bootstrap_bios.iso'),
|
||||
('ipxe.usb', 'iso/bootstrap_pxe_usb.img'),
|
||||
('ipxe.dsk', 'iso/bootstrap_pxe_floppy.img')):
|
||||
srcpath = os.path.join(srcdir, 'bin', fsrc)
|
||||
fname = os.path.basename(srcpath)
|
||||
destpath = os.path.join(destdir, fdest.format(f = fname))
|
||||
shutil.copy2(srcpath, destpath)
|
||||
for rom in rom_types:
|
||||
for s in rom_suffixes:
|
||||
fname = '{0}.{1}'.format(rom, s)
|
||||
fpath = os.path.join(srcdir, 'bin', fname)
|
||||
if os.path.isfile(fpath):
|
||||
shutil.copy2(fpath,
|
||||
os.path.join(destdir, 'roms', 'bootstrap_{0}'.format(fname)))
|
||||
# http://ipxe.org/howto/romburning
|
||||
# https://libvirt.org/formatdomain.html#elementsNICSROM
|
||||
if rom == '1af41000':
|
||||
os.symlink(fpath,
|
||||
os.path.join(destdir, 'roms', 'bootstrap_virtio.rom'))
|
||||
## EMBEDDED ##
|
||||
with open(os.path.join(logdir, 'all.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'all.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'all',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
for fsrc, fdest in (('undionly.kpxe', 'legacy/{f}'),
|
||||
('ipxe.iso', 'iso/bios.iso'),
|
||||
('ipxe.usb', 'iso/pxe_usb.img'),
|
||||
('ipxe.dsk', 'iso/pxe_floppy.img')):
|
||||
srcpath = os.path.join(srcdir, 'bin', fsrc)
|
||||
fname = os.path.basename(srcpath)
|
||||
destpath = os.path.join(destdir, fdest.format(f = fname))
|
||||
shutil.copy2(srcpath, destpath)
|
||||
for rom in rom_types:
|
||||
for s in rom_suffixes:
|
||||
fname = '{0}.{1}'.format(rom, s)
|
||||
fpath = os.path.join(srcdir, 'bin', fname)
|
||||
if os.path.isfile(fpath):
|
||||
shutil.copy2(fpath,
|
||||
os.path.join(destdir, 'roms', fname))
|
||||
if rom == '1af41000':
|
||||
os.symlink(fpath,
|
||||
os.path.join(destdir, 'roms', 'virtio.rom'))
|
||||
# DOS/MBR sector/HDD img.
|
||||
## BOOTSTRAP ##
|
||||
with open(os.path.join(logdir, 'bootstrap_hdd.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_hdd.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.hd',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
|
||||
os.path.join(destdir, 'iso', 'bootstrap_legacy_mbr.img'))
|
||||
## EMBEDDED ##
|
||||
with open(os.path.join(logdir, 'hdd.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'hdd.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.hd',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin/ipxe.hd'),
|
||||
os.path.join(destdir, 'iso', 'legacy_mbr.img'))
|
||||
# PXE loaders
|
||||
## BOOTSTRAP ##
|
||||
with open(os.path.join(logdir, 'bootstrap_loader.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_loader.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.pxe',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
|
||||
os.path.join(destdir, 'default', 'bootstrap_loader.pxe'))
|
||||
## EMBEDDED ##
|
||||
with open(os.path.join(logdir, 'loader.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'loader.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.pxe',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.pxe'),
|
||||
os.path.join(destdir, 'default', 'loader.pxe'))
|
||||
# EFI binaries and ISO images
|
||||
# These have to be done grouped because the eiso stuff APPARENTLY doesn't parse EMBED or lack thereof correctly?
|
||||
## BOOTSTRAP ##
|
||||
### EFI ###
|
||||
with open(os.path.join(logdir, 'bootstrap_efi.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_efi.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin-i386-efi/ipxe.efi',
|
||||
'bin-x86_64-efi/ipxe.efi',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', 'bootstrap_32.efi'))
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', 'bootstrap_64.efi'))
|
||||
### UEFI ISO ###
|
||||
with open(os.path.join(logdir, 'bootstrap_iso.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'bootstrap_iso.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.liso',
|
||||
'bin/ipxe.eiso',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'bootstrap.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
|
||||
os.path.join(destdir, 'iso', 'bootstrap_legacy.iso'))
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
|
||||
os.path.join(destdir, 'iso', 'bootstrap_uefi.iso'))
|
||||
## EMBEDDED ##
|
||||
### EFI ###
|
||||
with open(os.path.join(logdir, 'efi.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'efi.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin-i386-efi/ipxe.efi',
|
||||
'bin-x86_64-efi/ipxe.efi',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-i386-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', '32.efi'))
|
||||
shutil.copy2(os.path.join(srcdir, 'bin-x86_64-efi/ipxe.efi'),
|
||||
os.path.join(destdir, 'efi', '64.efi'))
|
||||
### UEFI ISO ###
|
||||
with open(os.path.join(logdir, 'iso.stderr'), 'wb') as stderr, \
|
||||
open(os.path.join(logdir, 'iso.stdout'), 'wb') as stdout:
|
||||
subprocess.run(['make',
|
||||
'bin/ipxe.liso',
|
||||
'bin/ipxe.eiso',
|
||||
'EMBED={0}'.format(os.path.join(configdir, 'chain-default.ipxe'))],
|
||||
stdout = stdout,
|
||||
stderr = stderr)
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.liso'),
|
||||
os.path.join(destdir, 'iso', 'legacy.iso'))
|
||||
os.rename(os.path.join(srcdir, 'bin/ipxe.eiso'),
|
||||
os.path.join(destdir, 'iso', 'uefi.iso'))
|
||||
return()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
56
builder/logger.py
Normal file
56
builder/logger.py
Normal file
@ -0,0 +1,56 @@
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
try:
|
||||
# https://www.freedesktop.org/software/systemd/python-systemd/journal.html#journalhandler-class
|
||||
from systemd import journal
|
||||
_has_journald = True
|
||||
except ImportError:
|
||||
_has_journald = False
|
||||
##
|
||||
from . import constants
|
||||
|
||||
|
||||
def preplog(logfile = None):
|
||||
if not logfile:
|
||||
logfile = constants.IPXE_DEFAULT_LOGFILE
|
||||
# Prep the log file.
|
||||
logfile = os.path.abspath(os.path.expanduser(logfile))
|
||||
os.makedirs(os.path.dirname(logfile), exist_ok = True, mode = 0o0700)
|
||||
if not os.path.isfile(logfile):
|
||||
with open(logfile, 'w') as fh:
|
||||
fh.write('')
|
||||
os.chmod(logfile, 0o0600)
|
||||
return(logfile)
|
||||
|
||||
|
||||
# And set up logging.
|
||||
_cfg_args = {'handlers': [],
|
||||
'level': logging.DEBUG}
|
||||
if _has_journald:
|
||||
# There were some weird changes somewhere along the line.
|
||||
try:
|
||||
# But it's *probably* this one.
|
||||
h = journal.JournalHandler()
|
||||
except AttributeError:
|
||||
h = journal.JournaldLogHandler()
|
||||
# Systemd includes times, so we don't need to.
|
||||
h.setFormatter(logging.Formatter(style = '{',
|
||||
fmt = ('{name}:{levelname}:{filename}:'
|
||||
'{funcName}:{lineno}: {message}')))
|
||||
_cfg_args['handlers'].append(h)
|
||||
|
||||
filehandler = logging.handlers.RotatingFileHandler(preplog(),
|
||||
encoding = 'utf8',
|
||||
# Disable rotating for now.
|
||||
# maxBytes = 50000000000,
|
||||
# backupCount = 30
|
||||
)
|
||||
filehandler.setFormatter(logging.Formatter(style = '{',
|
||||
fmt = ('{asctime}:'
|
||||
'{levelname}:{name}:{filename}:'
|
||||
'{funcName}:{lineno}: {message}')))
|
||||
_cfg_args['handlers'].append(filehandler)
|
||||
logging.basicConfig(**_cfg_args)
|
||||
logger = logging.getLogger('iPXE Builder')
|
||||
logger.info('Logging initialized.')
|
||||
134
builder/upstream.py
Normal file
134
builder/upstream.py
Normal file
@ -0,0 +1,134 @@
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tarfile
|
||||
import zipfile
|
||||
from urllib.parse import urlparse
|
||||
##
|
||||
import requests
|
||||
try:
|
||||
import git
|
||||
_has_git = True
|
||||
except ImportError:
|
||||
_has_git = False
|
||||
##
|
||||
from . import constants
|
||||
|
||||
|
||||
class _Upstream(object):
|
||||
def __init__(self, upstream_xml, *args, **kwargs):
|
||||
self.xml = upstream_xml
|
||||
self.uri = None
|
||||
self.proto = None
|
||||
self.host = None
|
||||
self.port = None
|
||||
self.path = None
|
||||
self.parsed_uri = None
|
||||
self.dest = os.path.abspath(os.path.expanduser(self.xml.find('dest').text))
|
||||
os.makedirs(os.path.dirname(self.dest), exist_ok = True, mode = 0o0750)
|
||||
|
||||
def parse_uri(self):
|
||||
self.parsed_uri = urlparse(self.uri)
|
||||
self.proto = self.parsed_uri.scheme.lower()
|
||||
self.host = self.parsed_uri.hostname.lower()
|
||||
self.port = int(getattr(self.parsed_uri, 'port', constants.DEF_PORTS[self.proto]))
|
||||
self.path = self.parsed_uri.path
|
||||
return(None)
|
||||
|
||||
|
||||
class Git(_Upstream):
|
||||
upstreamtype = 'git'
|
||||
|
||||
def __init__(self, upstream_xml, refresh = False, *args, **kwargs):
|
||||
if not _has_git:
|
||||
raise RuntimeError('The git module (GitPython) is not installed')
|
||||
super().__init__(upstream_xml, *args, **kwargs)
|
||||
self.repo = None
|
||||
self.remote = None
|
||||
self.ref = None
|
||||
self.refresh = refresh
|
||||
git_xml = self.xml.find('git')
|
||||
try:
|
||||
self.ref_name = git_xml.find('ref').text
|
||||
except AttributeError:
|
||||
self.ref_name = 'master'
|
||||
try:
|
||||
self.ref_type = git_xml.find('refType').text
|
||||
except AttributeError:
|
||||
self.ref_type = 'branch'
|
||||
self.uri = git_xml.find('uri').text
|
||||
self.parse_uri()
|
||||
|
||||
def fetch(self):
|
||||
if os.path.isdir(self.dest) and not self.refresh:
|
||||
self.repo = git.Repo(self.dest)
|
||||
elif os.path.isdir(self.dest):
|
||||
shutil.rmtree(self.dest)
|
||||
if not self.repo:
|
||||
self.repo = git.Repo.clone_from(self.uri, self.dest)
|
||||
for r in self.repo.remotes:
|
||||
if self.uri in r.urls:
|
||||
self.remote = r
|
||||
break
|
||||
if not self.remote:
|
||||
self.remote = self.repo.create_remote('bootbox_ipxe_upstream', self.uri)
|
||||
self.remote.pull()
|
||||
if self.ref_type in ('branch', 'head'):
|
||||
try:
|
||||
self.ref = self.repo.branches[self.ref_name]
|
||||
except IndexError:
|
||||
self.ref = self.repo.create_head(self.ref_name, self.remote.refs[self.ref_name])
|
||||
self.ref.set_tracking_branch(self.ref_name)
|
||||
self.ref.checkout()
|
||||
self.repo.head.reset(self.ref, working_tree = True)
|
||||
elif self.ref_type in ('tag', 'commit', 'rev'):
|
||||
if self.ref_type == 'tag':
|
||||
self.ref = self.repo.tags[self.ref_name]
|
||||
else:
|
||||
self.ref = self.repo.commit(rev = self.ref_name)
|
||||
self.repo.head.reset(self.ref, working_tree = True)
|
||||
self.repo.git.clean('-xdf')
|
||||
self.remote.pull()
|
||||
return(None)
|
||||
|
||||
|
||||
class Archive(_Upstream):
|
||||
upstreamtype = 'archive'
|
||||
|
||||
def __init__(self, upstream_xml, *args, **kwargs):
|
||||
super().__init__(upstream_xml, *args, **kwargs)
|
||||
self.uri = self.xml.find('archive').text
|
||||
self.parse_uri()
|
||||
self.suffix = re.sub(r'^.*\.(?P<suffix>(tar(\.((g|x)z|bz2?))?|t((g|x)z|bz2?)|zip))$',
|
||||
r'\g<suffix>',
|
||||
os.path.basename(self.path))
|
||||
self.req = None
|
||||
try:
|
||||
shutil.rmtree(self.dest)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.makedirs(self.dest, mode = 0o0750, exist_ok = True)
|
||||
|
||||
def fetch(self):
|
||||
self.req = requests.get(self.uri)
|
||||
buf = io.BytesIO(self.req.content)
|
||||
if not self.req.ok:
|
||||
raise RuntimeError('There was an error fetching the source')
|
||||
if self.suffix == 'zip':
|
||||
handler = zipfile.ZipFile(buf, 'r')
|
||||
else:
|
||||
handler = tarfile.open(fileobj = buf, mode = 'r')
|
||||
handler.extractall(self.dest)
|
||||
handler.close()
|
||||
return(None)
|
||||
|
||||
|
||||
def upstream_parser(upstream_xml):
|
||||
arc = upstream_xml.find('archive')
|
||||
gitrepo = upstream_xml.find('git')
|
||||
if arc:
|
||||
return(Archive(upstream_xml))
|
||||
elif gitrepo:
|
||||
return(Git(upstream_xml))
|
||||
return(None)
|
||||
Loading…
Reference in New Issue
Block a user