r00t2/bdisk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

safety checkin... getting super close

Browse Source
This commit is contained in:
brent s. 2016-11-28 02:56:15 -05:00
parent 0102ca26c3
commit 736457a6e0
23 changed files with 575 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -25,6 +25,7 @@
/logs
*.swp
*.lck
*~
/extrasrc

# You should really generate local copies of these, as they're pretty private.

2
TODO
View File

@ -1,2 +0,0 @@
- templating via jinja2:
-- for syslinux/isolinux configs

21
bdisk/bchroot.py
View File

@ -29,35 +29,35 @@ def chroot(chrootdir, chroot_hostname, cmd = '/root/pre-build.sh'):
mounts.append(m.mountpoint)
# mount the chrootdir... onto itself. as a bind mount. it's so stupid, i know. see https://bugs.archlinux.org/task/46169
if chrootdir not in mounts:
subprocess.call(['mount', '--bind', chrootdir, chrootdir])
subprocess.call(['/bin/mount', '--bind', chrootdir, chrootdir])
### The following mountpoints don't seem to mount properly with pychroot. save it for v3.n+1. TODO. ###
# bind-mount so we can resolve things inside
if (chrootdir + '/etc/resolv.conf') not in mounts:
subprocess.call(['mount', '--bind', '-o', 'ro', '/etc/resolv.conf', chrootdir + '/etc/resolv.conf'])
subprocess.call(['/bin/mount', '--bind', '-o', 'ro', '/etc/resolv.conf', chrootdir + '/etc/resolv.conf'])
# mount -t proc to chrootdir + '/proc' here
if (chrootdir + '/proc') not in mounts:
subprocess.call(['mount', '-t', 'proc', '-o', 'nosuid,noexec,nodev', 'proc', chrootdir + '/proc'])
subprocess.call(['/bin/mount', '-t', 'proc', '-o', 'nosuid,noexec,nodev', 'proc', chrootdir + '/proc'])
# rbind mount /sys to chrootdir + '/sys' here
if (chrootdir + '/sys') not in mounts:
subprocess.call(['mount', '-t', 'sysfs', '-o', 'nosuid,noexec,nodev,ro', 'sys', chrootdir + '/sys'])
subprocess.call(['/bin/mount', '-t', 'sysfs', '-o', 'nosuid,noexec,nodev,ro', 'sys', chrootdir + '/sys'])
# mount the efivars in the chroot if it exists on the host. i mean, why not?
if '/sys/firmware/efi/efivars' in mounts:
if (chrootdir + '/sys/firmware/efi/efivars') not in mounts:
subprocess.call(['mount', '-t', 'efivarfs', '-o', 'nosuid,noexec,nodev', 'efivarfs', chrootdir + '/sys/firmware/efi/efivars'])
subprocess.call(['/bin/mount', '-t', 'efivarfs', '-o', 'nosuid,noexec,nodev', 'efivarfs', chrootdir + '/sys/firmware/efi/efivars'])
# rbind mount /dev to chrootdir + '/dev' here
if (chrootdir + '/dev') not in mounts:
subprocess.call(['mount', '-t', 'devtmpfs', '-o', 'mode=0755,nosuid', 'udev', chrootdir + '/dev'])
subprocess.call(['/bin/mount', '-t', 'devtmpfs', '-o', 'mode=0755,nosuid', 'udev', chrootdir + '/dev'])
if (chrootdir + '/dev/pts') not in mounts:
subprocess.call(['mount', '-t', 'devpts', '-o', 'mode=0620,gid=5,nosuid,noexec', 'devpts', chrootdir + '/dev/pts'])
subprocess.call(['/bin/mount', '-t', 'devpts', '-o', 'mode=0620,gid=5,nosuid,noexec', 'devpts', chrootdir + '/dev/pts'])
if '/dev/shm' in mounts:
if (chrootdir + '/dev/shm') not in mounts:
subprocess.call(['mount', '-t', 'tmpfs', '-o', 'mode=1777,nosuid,nodev', 'shm', chrootdir + '/dev/shm'])
subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'mode=1777,nosuid,nodev', 'shm', chrootdir + '/dev/shm'])
if '/run' in mounts:
if (chrootdir + '/run') not in mounts:
subprocess.call(['mount', '-t', 'tmpfs', '-o', 'nosuid,nodev,mode=0755', 'run', chrootdir + '/run'])
subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'nosuid,nodev,mode=0755', 'run', chrootdir + '/run'])
if '/tmp' in mounts:
if (chrootdir + '/tmp') not in mounts:
subprocess.call(['mount', '-t', 'tmpfs', '-o', 'mode=1777,strictatime,nodev,nosuid', 'tmp', chrootdir + '/tmp'])
subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'mode=1777,strictatime,nodev,nosuid', 'tmp', chrootdir + '/tmp'])

print("Now performing '{0}' in chroot for {1}...".format(cmd, chrootdir))
print("You can view the progress via:\n\n\ttail -f {0}/var/log/chroot_install.log\n".format(chrootdir))
@ -70,5 +70,4 @@ def chroot(chrootdir, chroot_hostname, cmd = '/root/pre-build.sh'):
return(chrootdir)

def chrootUnmount(chrootdir):
# TODO: https://github.com/pkgcore/pychroot/issues/22 try to do this more pythonically. then we can remove subprocess
subprocess.call(['umount', '-lR', chrootdir])

9
bdisk/bdisk.py
View File

@ -5,21 +5,18 @@ import bchroot
import build

# we need to:
# 8.) build.chrootClean (TODO) see jenny_craig in old bdisk. i can *probably* do this within the chroot for the most part as part of pre-build.sh
# 9.) build.genImg (TODO)- build the squashed image, etc. see will_it_blend in old bdisk
# 10.) build.genUEFI (TODO)- build the uefi binary/bootloading. see stuffy in old bdisk
# 11.) build.genISO (TODO)- build the .iso file (full boot). see yo_dj in old bdisk
# 9.5) copy the files also in the same script. after the commented-out mtree-generation
#
# we also need to figure out how to implement "mentos" (old bdisk) like functionality, letting us reuse an existing chroot install if possible to save time for future builds.
# if not, though, it's no big deal.
if __name__ == '__main__':
# TODO: config for chrootdir, dlpath
conf = host.parseConfig(host.getConfig())[1]
prep.dirChk(conf)
prep.buildChroot(conf['build'])
prep.prepChroot(conf['build']['basedir'] + '/extra/templates', conf['build'], conf['bdisk'])
prep.prepChroot(conf['build'], conf['bdisk'])
arch = conf['build']['arch']
for a in arch:
bchroot.chroot(conf['build']['chrootdir'] + '/root.' + a, 'bdisk.square-r00t.net')
bchroot.chrootUnmount(conf['build']['chrootdir'] + '/root.' + a)
#build.chrootClean(conf['build'])
prep.postChroot(conf['build'])

356
bdisk/build.py
View File

@ -2,109 +2,271 @@ import os
import tarfile
import shutil
import glob
import subprocess
import hashlib
import jinja2
from urllib.request import urlopen


def chrootClean(build):
chrootdir = build['chrootdir']
def genImg(build):
arch = build['arch']
backupdir = build['dlpath'] + '/' + 'bak'
os.makedirs(backupdir, exist_ok = True)
## Save some stuff that needs to be retained.
# Compress the pacman cache.
chrootdir = build['chrootdir']
archboot = build['archboot']
hashes = {}
hashes['sha256'] = {}
hashes['md5'] = {}
for a in arch:
os.makedirs(chrootdir + '/root.' + a + '/usr/local/pacman', exist_ok = True)
tarball = chrootdir + '/root.' + a + '/usr/local/pacman/pacman.db.tar.xz'
dbdir = chrootdir + '/root.' + a + '/var/lib/pacman/local'
print("Now cleaning {0}/root.{1}. Please wait...".format(chrootdir, a))
if os.path.isfile(tarball):
os.remove(tarball)
with tarfile.open(name = tarball, mode = 'w:xz') as tar: # if this balks, try x:xz
tar.add(dbdir, arcname = os.path.basename(dbdir))
# Cut out the fat
# The following are intended as "overrides" of the paths we'll be deleting.
backup = ['/var/lib/pacman/local',
'/usr/share/locale/locale.alias',
'/usr/share/zoneinfo/EST5EDT',
'/usr/share/zoneinfo/UTC',
'/usr/share/locale/en',
'/usr/share/locale/en_US',
'/usr/share/locale/en_GB']
# And these are what we remove.
delete = ['/usr/share/locale/',
'/var/cache/pacman/',
'/var/cache/pkgfile/',
'/var/cache/apacman/pkg/',
'/var/lib/pacman/',
'/var/abs/local/yaourtbuild/',
'/usr/share/zoneinfo',
'/root/.gnupg',
'/tmp/',
'/var/tmp/',
'/var/abs/',
'/run/',
'/boot/',
'/usr/src/',
'/var/log/',
'/.git']
delete['files'] = ['/root/.bash_history',
'/root/apacman*',
'/root/iso.pkgs*',
'/root/packages.*',
'/root/pre-build.sh',
'/root/.viminfo',
'/root/.bashrc']
# First we backup files. We don't need to create backup['dirs']
# since they should be empty. If not, they go in backup['files'].
for f in backup['files']:
#os.makedirs(backupdir + '/root.' + a + os.path.dirname(f), exist_ok = True)
#shutil.copy2(chrootdir + '/root.' + a + f, backupdir + '/root.' + a + f)
for root, dirs, files in os.walk(f):
for item in files:
src_path = os.path.join(root, item)
dst_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
if os.path.exists(dst_path):
if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime:
shutil.copy2(src_path, dst_path)
# Create the squashfs image
airoot = archboot + '/' + a + '/'
squashimg = airoot + 'airootfs.sfs'
os.makedirs(airoot, exist_ok = True)
print("Generating squashed filesystem image for {0}. Please wait...".format(chrootdir))
cmd = ['/usr/bin/squashfs-tools', chrootdir, squashimg, '-noappend', '-comp', 'xz']
proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, bufsize = 1)
#for line in iter(proc.stdout.readline, b''):
for line in iter(proc.stdout.readline, ''):
print(line)
p.stdout.close()
p.wait()
# Generate the checksum files
print("Now generating SHA256 and MD5 hash checksum files for {0}. Please wait...".format(squashimg))
hashes['sha256'][a] = hashlib.sha256()
hashes['md5'][a] = hashlib.md5()
with open(squashimg, 'rb') as f:
while True:
stream = f.read(65536) # 64kb chunks
if not stream:
break
# NOTE: these items are hashlib objects, NOT strings!
hashes['sha256'][a].update(stream)
hashes['md5'][a].update(stream)
with open(airoot + 'airootfs.sha256', 'w+') as f:
f.write("{0} airootfs.sfs".format(hashes['sha256'][a].hexdigest()))
with open(airoot + 'airootfs.md5', 'w+') as f:
f.write("{0} airootfs.sfs".format(hashes['md5'][a].hexdigest()))


def genUEFI(build, bdisk):
arch = build['arch']
# 32-bit EFI implementations are nigh nonexistant.
# We don't really need to worry about them.
# Plus there's always multiarch.
# I can probably do this better with a dict... TODO.
if 'x86_64' in arch:
os.makedirs(tempdir + '/EFI/boot', exist_ok = True)
tempdir = build['tempdir']
basedir = build['basedir']
templates_dir = build['basedir'] + '/extra/templates'
efiboot_img = tempdir + '/EFI/' + bdisk['name'] + '/efiboot.img'
## Download the EFI shells if we don't have them.
# For UEFI 2.3+ (http://sourceforge.net/apps/mediawiki/tianocore/index.php?title=UEFI_Shell)
if not os.path.isfile(tempdir + '/EFI/shellx64_v2.efi'):
shell2_path = tempdir + '/EFI/shellx64_v2.efi'
print("You are missing {0}. We'll download it for you.".format(shell2_path))
shell2_url = 'https://github.com/tianocore/edk2/blob/master/ShellBinPkg/UefiShell/X64/Shell.efi?raw=true'
shell2_fetch = urlopen(shell2_url)
with open(shell2_path, 'wb+') as dl:
dl.write(shell2_fetch.read())
shell2_fetch.close()
# Shell for older versions (http://sourceforge.net/apps/mediawiki/tianocore/index.php?title=Efi-shell)
# TODO: is there an Arch package for this? can we just install that in the chroot and copy the shell binaries?
if not os.path.isfile(tempdir + '/EFI/shellx64_v1.efi'):
shell1_path = tempdir + '/EFI/shellx64_v1.efi'
print("You are missing {0}. We'll download it for you.".format(shell1_path))
shell1_url = 'https://github.com/tianocore/edk2/blob/master/EdkShellBinPkg/FullShell/X64/Shell_Full.efi?raw=true'
shell1_fetch = urlopen(shell1_url)
with open(shell1_path, 'wb+') as dl:
dl.write(shell1_fetch.read())
shell1_fetch.close()
print("Now configuring UEFI bootloading...")
## But wait! That's not all! We need more binaries.
# http://blog.hansenpartnership.com/linux-foundation-secure-boot-system-released/
shim_url = 'http://blog.hansenpartnership.com/wp-uploads/2013/'
for f in ('bootx64.efi', 'HashTool.efi'):
if not os.path.isfile(tempdir + '/EFI/boot/' + f):
url = shim_url + f
url_fetch = urlopen(url)
with open(tempdir + '/EFI/boot/' + f) as dl:
dl.write(url_fetch.read())
url_fetch.close()
# And we also need the systemd efi bootloader.
if os.path.isfile(tempdir + '/EFI/boot/loader.efi'):
os.remove(tempdir + '/EFI/boot/loader.efi')
shutil.copy2(chrootdir + '/root.x86_64/usr/lib/systemd/boot/efi/systemd-bootx64.efi', tempdir + '/EFI/boot/loader.efi')
# And the accompanying configs for the systemd efi bootloader, too.
loader = jinja2.FileSystemLoader(templates_dir)
env = jinja2.Environment(loader = loader)
os.makedirs(tempdir + '/loader/entries', exist_ok = True)
for t in ('loader', 'ram', 'base', 'uefi2', 'uefi1'):
if t == 'base':
fname = bdisk['uxname'] + '.conf'
elif ('uefi1', 'uefi2') in t:
fname = t + '.conf'
else:
shutil.copy2(src_path, dst_path)
for item in dirs:
src_path = os.path.join(root, item)
dst_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
os.makedirs(dst_path, exist_ok = True)
# Now we delete the above.
for f in delete['files']:
for x in glob.glob(chrootdir + '/root.' + a + f):
os.remove(x)
for d in delete['dirs']:
for x in glob.glob(chrootdir + '/root.' + a + d):
#os.remove(x)
shutil.rmtree(x)
# And restore the dirs/files
for d in backup['dirs']:
os.makedirs(chrootdir + '/root.' + a + d, exist_ok = True)
for f in backup['files']:
#os.makedirs(chrootdir + '/root.' + a + os.path.dirname(f), exist_ok = True)
#shutil.copy2(backupdir + '/root.' + a + f, chrootdir + '/root.' + a + f)
for root, dirs, files in os.walk(f):
for item in files:
src_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
dst_path = os.path.join(root, item)
if os.path.exists(dst_path):
if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime:
shutil.copy2(src_path, dst_path)
fname = bdisk['uxname'] + '_' + t + '.conf'
if t == 'loader':
tplpath = tempdir + '/loader/'
fname = 'loader.conf' # we change the var from above because it's an oddball.
else:
shutil.copy2(src_path, dst_path)
for item in dirs:
src_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
dst_path = os.path.join(root, item)
os.makedirs(dst_path, exist_ok = True)
#shutil.rmtree(backupdir)
tplpath = tempdir + '/loader/entries/'
tpl = env.get_template(t)
tpl_out = tpl.render(build = build, bdisk = bdisk)
with open(tplpath + fname, "w+") as f:
f.write(tpl_out)
# And we need to get filesizes (in bytes) for everything we need to include in the ESP.
# This is more important than it looks.
sizetotal = 786432 # we start with 768KB and add to it for wiggle room
sizefiles = ['/boot/' + bdisk['uxname'] + '.64.img',
'/EFI/boot/bootx64.efi',
'/EFI/boot/loader.efi',
'/EFI/boot/HashTool.efi',
'/EFI/shellx64_v1.efi',
'/EFI/shellx64_v2.efi']
for i in sizefiles:
sizetotal += os.path.getsize(tempdir + i)
# Loader configs
for (path, dirs, files) in os.walk(tempdir + '/loader/'):
for file in files:
fname = os.path.join(path, file)
sizetotal += os.path.getsize(fname)
# And now we create the file...
print("Now creating a {0} bytes EFI ESP image at {1}. Please wait...".format(sizetotal, efiboot_img))
if os.path.isfile(efiboot_img):
os.remove(efiboot_img)
with open(efiboot_img, 'w+') as f:
f.truncate(sizetotal)
cmd = ['/sbin/mkfs.vfat', '-F', '32', '-n', bdisk['name'] + '_EFI', efiboot_img]
subprocess.call(cmd)
cmd = ['/bin/mount', efiboot_img, build['mountpt']]
subprocess.call(cmd)
os.makedirs(build['mountpt'] + '/' + bdisk['name'])
os.makedirs(build['mountpt'] + '/EFI/boot')
os.makedirs(build['mountpt'] + '/loader/entries')
# Ready for some deja vu? This is because it uses an embedded version as well for hybrid ISO.
# I think.
# TODO: just move this to a function instead, with "efi" as a param and change
# the templates to use "if efi == 'yes'" instead.
# function should set the "installation" path for the conf as well based on the value of efi
# parameter.
loader = jinja2.FileSystemLoader(templates_dir)
env = jinja2.Environment(loader = loader)
os.makedirs(build['mountpt'] + 'loader/entries', exist_ok = True)
for t in ('loader', 'ram', 'base', 'uefi2', 'uefi1'):
if t == 'base':
fname = bdisk['uxname'] + '.conf'
elif ('uefi1', 'uefi2') in t:
fname = t + '.conf'
else:
fname = bdisk['uxname'] + '_' + t + '.conf'
if t == 'loader':
tplpath = build['mountpt'] + '/loader/'
fname = 'loader.conf' # we change the var from above because it's an oddball.
else:
tplpath = build['mountpt'] + '/loader/entries/'
tpl = env.get_template(t)
tpl_out = tpl.render(build = build, bdisk = bdisk, efi = 'yes')
with open(tplpath + fname, "w+") as f:
f.write(tpl_out)
for x in ('bootx64.efi', 'HashTool.efi', 'loader.efi', 'shellx64_v1.efi', 'shellx64_v2.efi'):
y = tempdir + '/EFI/boot/' + x
z = mountpt + '/EFI/boot/' + x
if os.path.isfile(z):
os.remove(z)
shutil.copy(y, z)
cmd = ['/bin/umount', mountpt]
subprocess.call(cmd)
return(efiboot_img)

def genImg():
pass
def genISO(conf):
build = conf['build']
bdisk = conf['bdisk']
archboot = build['archboot']
tempdir = build['tempdir']
builddir = tempdir + '/' + bdisk['name']
extradir = build['basedir'] + '/extra/'
# arch[0] is safe to use, even if multiarch, because the only cases when it'd be ambiguous
# is when x86_64 is specifically set to [0]. See host.py's parseConfig().
syslinuxdir = build['chrootdir'] + '/root.' + arch[0] + '/usr/lib/syslinux/'
sysl_tmp = tempdir + '/isolinux/'
ver = build['ver']
isofile = '{0}-{1}.iso'.format(bdisk['uxname'], bdisk['ver'])
isopath = build['isodir'] + '/' + isofile
arch = build['arch']
# In case we're building a single-arch ISO...
if len(arch) == 1:
isolinux_cfg = extradir + 'templates/BIOS/isolinux.cfg.arch.j2'
if arch[0] == 'i686':
bitness = '32'
elif arch[0] == 'x86_64':
bitness = '64'
if build['ipxe']:
ipxe = conf['ipxe']
if ipxe['iso']:
minifile = '{0}-{1}-mini.iso'.format(bdisk['uxname'], bdisk['ver'])
minipath = build['isodir'] + '/' + minifile
if ipxe['usb']:
usbfile = '{0}-{1}-mini.usb.img'.format(bdisk['uxname'], bdisk['ver'])
minipath = build['isodir'] + '/' + usbfile
else:
isolinux_cfg = extradir + 'templates/BIOS/isolinux.cfg.multi.j2'
bitness = False
if os.path.isfile(isopath):
os.remove(isopath)
if archboot != tempdir + '/' + bdisk['name']: # best to use static concat here...
if os.path.isdir(builddir):
shutil.rmtree(builddir, ignore_errors = True)
shutil.copytree(archboot, builddir)
# Copy isolinux files
print("Now staging some files for ISO preparation. Please wait...")
isolinux_files = ['isolinux.bin',
'vesamenu.c32',
'linux.c32',
'reboot.c32']
# TODO: implement debugging mode in bdisk
#if debug:
# isolinux_files[0] = 'isolinux-debug.bin'
os.makedirs(sysl_tmp, exist_ok = True)
for f in isolinux_files:
if os.path.isfile(sysl_tmp + f):
os.remove(sysl_tmp + f)
shutil.copy2(syslinuxdir + f, sysl_tmp + f)
ifisolinux_files = ['ldlinux.c32',
'libcom32.c32',
'libutil.c32',
'ifcpu64.c32']
for f in ifisolinux_files:
if os.path.isfile(sysl_tmp + f):
os.remove(sysl_tmp + f)
shutil.copy2(syslinuxdir + f, sysl_tmp + f)
# And we need to build the ISO!
print("Now generating the full ISO at {0}. Please wait.".format(isopath))
cmd = ['/usr/bin/xorriso',
'-as', 'mkisofs',
'-iso-level', '3',
'-full-iso9660-filenames',
'-volid', bdisk['name'],
'-appid', bdisk['desc'],
'-publisher', bdisk['dev'],
'-preparer', 'prepared by ' + bdisk['dev'],
'-eltorito-boot', 'isolinux/isolinux.bin',
'-eltorito-catalog', 'isolinux/boot.cat',
'-no-emul-boot',
'-boot-load-size', '4',
'-boot-info-table',
'-isohybrid-mbr', syslinuxdir + 'isohdpfx.bin',
'-eltorito-alt-boot',
'-e', 'EFI/' + bdisk['name'] + '/efiboot.img',
'-no-emul-boot',
'-isohybrid-gpt-basdat',
'-output', isopath,
tempdir]
proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, bufsize = 1)
#for line in iter(proc.stdout.readline, b''):
for line in iter(proc.stdout.readline, ''):
print(line)
p.stdout.close()
p.wait()

def genUEFI():
pass

def genISO():
def cleanUp():
# TODO: clear out all of tempdir?
pass

13
bdisk/ipxe.py Executable file
View File

@ -0,0 +1,13 @@
import os
import shutil
import jinja2
import gitpython


def buildIPXE(conf):
build = conf['build']
bdisk = conf['bdisk']
ipxe = conf['ipxe']
templates_dir = build['basedir'] + '/extra/templates'
patches_dir = build['basedir'] +
pass

52
bdisk/prep.py
View File

@ -12,6 +12,7 @@ import datetime
from urllib.request import urlopen
import host # bdisk.host


def dirChk(config_dict):
# Make dirs if they don't exist
for d in ('archboot', 'isodir', 'mountpt', 'srcdir', 'tempdir'):
@ -143,20 +144,21 @@ def buildChroot(build):
os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0)
# and copy over the files. again, chown to root.
for file in prebuild_overlay['files']:
shutil.copy2(extradir + '/pre-build.d/' + file, chrootdir + '/root.' + a + '/' + file)
shutil.copy2(extradir + '/pre-build.d/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0)
# do the same for arch-specific stuff.
for dir in prebuild_arch_overlay[a]['dirs']:
os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0)
for file in prebuild_arch_overlay[a]['files']:
shutil.copy2(extradir + '/pre-build.d/' + a + '/' + file, chrootdir + '/root.' + a + '/' + file)
shutil.copy2(extradir + '/pre-build.d/' + a + '/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0)

def prepChroot(templates_dir, build, bdisk):
def prepChroot(build, bdisk):
chrootdir = build['chrootdir']
arch = build['arch']
bdisk_repo_dir = build['basedir']
templates_dir = bdisk_repo_dir + '/extra/templates'
build = {}
# let's prep some variables to write out the version info.txt
# get the git tag and short commit hash
@ -174,8 +176,50 @@ def prepChroot(templates_dir, build, bdisk):
loader = jinja2.FileSystemLoader(templates_dir)
env = jinja2.Environment(loader = loader)
tpl = env.get_template('VERSION_INFO.txt.j2')
tpl_out = tpl.render(build = build, hostname = hostname)
tpl_out = tpl.render(build = build, hostname = host.getHostname())
for a in arch:
with open(chrootdir + '/root.' + a + '/root/VERSION_INFO.txt', "w+") as f:
f.write(tpl_out)
return(build)

def postChroot(build):
dlpath = build['dlpath']
chrootdir = build['chrootdir']
arch = build['arch']
overdir = build['basedir'] + '/overlay/'
postbuild_overlay = {}
postbuild_arch_overlay = {}
for x in arch:
postbuild_arch_overlay[x] = {}
for y in ['files', 'dirs']:
postbuild_overlay[y] = []
postbuild_arch_overlay[x][y] = []
for path, dirs, files in os.walk(overdir):
postbuild_overlay['dirs'].append(path + '/')
for file in files:
postbuild_overlay['files'].append(os.path.join(path, file))
for x in postbuild_overlay.keys():
postbuild_overlay[x][:] = [re.sub('^' + overdir, '', s) for s in postbuild_overlay[x]]
postbuild_overlay[x] = list(filter(None, postbuild_overlay[x]))
for y in postbuild_arch_overlay.keys():
postbuild_arch_overlay[y][x][:] = [i for i in postbuild_overlay[x] if i.startswith(y)]
postbuild_arch_overlay[y][x][:] = [re.sub('^' + y + '/', '', s) for s in postbuild_arch_overlay[y][x]]
postbuild_arch_overlay[y][x] = list(filter(None, postbuild_arch_overlay[y][x]))
postbuild_overlay[x][:] = [y for y in postbuild_overlay[x] if not y.startswith(('x86_64','i686'))]
postbuild_overlay['dirs'].remove('/')
# create the dir structure. these should almost definitely be owned by root.
for a in arch:
for dir in postbuild_overlay['dirs']:
os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0, follow_symlinks = False)
# and copy over the files. again, chown to root.
for file in postbuild_overlay['files']:
shutil.copy2(overdir + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0, follow_symlinks = False)
# do the same for arch-specific stuff.
for dir in postbuild_arch_overlay[a]['dirs']:
os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0, follow_symlinks = False)
for file in postbuild_arch_overlay[a]['files']:
shutil.copy2(overdir + a + '/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0, follow_symlinks = False)

12
docs/CREDITS Normal file
View File

@ -0,0 +1,12 @@
iPXE:
Thanks to "eworm" for his work on the AUR iPXE-git package:
https://aur.archlinux.org/packages/ipxe-git/

and specifically the following patches:
http://www.eworm.de/download/linux/ipxe-0001-git-version.patch
http://www.eworm.de/download/linux/ipxe-0002-banner.patch
http://www.eworm.de/download/linux/ipxe-0003-iso-efi.patch
http://www.eworm.de/download/linux/ipxe-0004-fix-no-pie-workaround.patch


thanks to jthan, even though he drives me batty sometimes.

1
docs/THANKS Symbolic link
View File

@ -0,0 +1 @@
CREDITS

37
docs/TODO
View File

@ -1,3 +1,8 @@
## Missing v2.x functionality ##
-i_am_a_racecar optimizations
-see also: grep -HnR bdisk/*.py
-TFTP, HTTP, RSYNC, git

## General ##

-include benchmarking
@ -5,8 +10,7 @@
-- https://code.google.com/p/byte-unixbench/
-- https://github.com/akopytov/sysbench
-- (http://blog.due.io/2014/linode-digitalocean-and-vultr-comparison/ etc.)
-package in AUR
-base rewrite in python. pyalpm may come in handy here.
-implement pyalpm to decreate dependency on chroot pacman-ing?


## NETWORKING ##
@ -23,28 +27,27 @@

## Building ##

-GUMMIBOOT IS GONE FROM THE REPOS. I could repackage it, but better to just see what the hell archiso's doing.
-WISH: Better logging[0]
-WISH: signing for secureboot releases (PreLoader and gummiboot handle this okay, but require manual intervention
-use manual chrooting functions ONLY if distro not detected as arch. if /usr/bin/systemd-nspawn exists, use that instead
--does arch-chroot work across all distros? see https://wiki.archlinux.org/index.php/Install_bundled_32-bit_system_in_Arch64 and https://wiki.archlinux.org/index.php/Chroot
--i think this might be unnecessary. testing across other major distros is necessary, but i think i can just use the chroot'd arch-chroot
-tweak build.conf (and build.conf.sample) to source the pwd and set as BASEDIR ***if*** the project resources are present in pwd, otherwise throw warning
--this is half-done;PWD is currently used by default.
-does gummiboot? loader? wtfever it's called support splash backgrounds? can i implement that differently somehow?
-WISH: signing for secureboot releases (PreLoader and loader.efi handle this okay, but require manual intervention)
-does loader.efi support splash backgrounds? can i implement that differently somehow?
--yes, see e.g. https://www.reddit.com/r/archlinux/comments/3bwgf0/where_put_the_splasharchbmp_to_splash_screen_boot/
-strip out/remove unnecessary and orphan packages (e.g. gcc, make, automake, etc.)
-incorporate iPXE tweaks:
--http://ipxe.org/crypto
--http://ipxe.org/cmd/imgtrust
--http://ipxe.org/cmd/imgverify
--enable use of custom CA/self-signed certs for HTTPS etc. DONE, partially. need to incorporate codesign certs/keys. routines, conf variables
--enable use of custom CA/self-signed certs for HTTPS etc.
--signed kernel and initrd for ipxe:
---#imgtrust --permanent
---#imgverify vmlinuz path/to/vmlinuz.sig
---#imgverify initrd path/to/initrd.sig
---DONE, partially. need to incorporate codesign certs/keys. routines, conf variables
-enable mirror= kernel commandline.
--if mirror_(NAME) is present, use that as repo name.
--if it starts with /, treat as mirrorlist (Include); otherwise use Server =
--if it has mirror_SIG-X, set signature options e.g. _SIG-N would be "SigLevel = Never"
-iPXE background support. sed -rf "${BASEDIR}/src/ipxe_local/script.sed" ${SRCDIR}/ipxe/src/config/general.h ; sed -rf "${BASEDIR}/src/ipxe_local/script2.sed" ${SRCDIR}/ipxe/src/config/console.h
--note that iPXE VESAFB console is not (yet) supported in EFI, so this is on hold.
--note that iPXE VESAFB console is not (yet) supported in EFI, so this is on hold. check into this to see if it has changed.

## Split into Separate Tools CD ##

@ -57,12 +60,4 @@ __________________________________________________________
FOOTNOTES:


[0] I'd really like to implement the following in build.conf; like:
http://forums.fedoraforum.org/showthread.php?t=275743
# The following is the setting for "verbosity". A more accurate way of saying it is how output should be handled.
# Note that for it to be properly parsed, it MUST be in the form of a linear array (e.g. VAR=(1 2 3) ).
# '| tee -a ${BASEDIR}/logs/${FUNCNAME}.$(date +%s)' means "display output for STDOUT and STDERR, and also log STDOUT to logs/<function name>.EPOCH_TIME"
# '2>&1 /dev/null' means "hide STDOUT and STDERR, no logging"
# '>> ${BASEDIR}/logs/${FUNCNAME}.$(date +%s) 2>&1' means "log both STDOUT and STDERR to logs/<function name>.EPOCH_TIME, no output"
# '>> ${BASEDIR}/logs/${FUNCNAME}.$(date +%s)' means "log STDOUT to logs/<function name>.EPOCH_TIME, display (but don't log) STDERR)"
# '' means "no logging; display both STDOUT and STDERR"
[0] Debugging mode would probably suffice for logging? Maybe? There are plenty of pythonic ways to do logging too.

5
extra/pre-build.d/root/pre-build.sh
View File

@ -145,3 +145,8 @@ then
apacman --gendb
cleanPacorigs
fi
# Cleanup
yes | pacman -Scc
rm -f /root/.bash_history
rm -f /root/.viminfo


73
extra/templates/BIOS/isolinux.cfg.arch.j2 Normal file
View File

@ -0,0 +1,73 @@
UI vesamenu.c32
DEFAULT check
PROMPT 0
TIMEOUT 50
MENU HIDDEN
ONTIMEOUT {{ bdisk['uxname'] }}_ram_{{ bitness }}
MENU TABMSG Press [TAB] to edit options
#MENU TITLE {{ bdisk['pname'] }} (ISO edition)
MENU ROWS 16
MENU TIMEOUTROW 22
MENU TABMSGROW 24
MENU CMDLINEROW 24
MENU HELPMSGROW 26
MENU WIDTH 78
MENU MARGIN 6
MENU IMMEDIATE
# http://www.colorpicker.com/
MENU color border 0 #00000000 #00000000 none
MENU color title 0 #FFF5B800 #00000000 std
MENU color sel 7;37;40 #FF000000 #FFFFFFFF all
MENU color hotsel 1;7;37;40 #FFFF0000 #FFC0C0C0 all
MENU color hotkey 1;7;37;40 #FF0000CC #FFC0C0C0 all
MENU color tabmsg 1;31;40 #FF808080 #00000000 std
MENU color help 1;31;40 #FFFFFFFF #FF000000 none
MENU color timeout_msg 0 #FFFFB300 #00000000 none
MENU color timeout 0 #FFFF0000 #FF000000 none
MENU color cmdline 0 #FFFFFFFF #FF000000 none
MENU color cmdmark 1;36;40 #C000FFFF #FF000000 std
MENU color scrollbar 30;44 #FF00FF00 #FF000000 std
MENU color msg07 0 #FF000000 #00FFFFFF none
MENU BACKGROUND /{{ bdisk['uxname'] }}.png

LABEL local_override
MENU LABEL Local ^Boot
localboot 0
TEXT HELP
Boot from the local system instead.
ENDTEXT

LABEL reboot
MENU LABEL ^Reboot
COM32 reboot.c32
TEXT HELP
Reboot the machine
ENDTEXT

MENU SEPARATOR

MENU BEGIN {{ bitness }}BIT
MENU LABEL ^1) {{ bitness }}-Bit ...
ONTIMEOUT {{ bdisk['uxname'] }}_{{ bitness }}
LABEL {{ bdisk['uxname'] }}_ram_{{ bitness }}
MENU LABEL ^1) {{ bdisk['pname'] }} (run from RAM)
LINUX /boot/{{ bdisk['uxname'] }}.{{ bitness }}.kern
INITRD /boot/{{ bdisk['uxname'] }}.{{ bitness }}.img
APPEND copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
TEXT HELP
{{ bitness }}-bit, run from RAM
ENDTEXT
MENU DEFAULT

LABEL {{ bdisk['uxname'] }}_{{ bitness }}
MENU LABEL ^1) {{ bdisk['pname'] }} (Default)
LINUX /boot/{{ bdisk['uxname'] }}.{{ bitness }}.kern
INITRD /boot/{{ bdisk['uxname'] }}.{{ bitness }}.img
APPEND archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
TEXT HELP
Same as the above, except run directly from the CD-
don't copy the image to RAM. (Best for lower-memory boxes)
ENDTEXT

MENU END

110
extra/templates/BIOS/isolinux.cfg.multi.j2 Normal file
View File

@ -0,0 +1,110 @@
UI vesamenu.c32
DEFAULT check
PROMPT 0
TIMEOUT 50
MENU HIDDEN
#ONTIMEOUT {{ bdisk['uxname'] }}_ram
ONTIMEOUT check
MENU TABMSG Press [TAB] to edit options
#MENU TITLE {{ bdisk['pname'] }} (ISO edition)
MENU ROWS 16
MENU TIMEOUTROW 22
MENU TABMSGROW 24
MENU CMDLINEROW 24
MENU HELPMSGROW 26
MENU WIDTH 78
MENU MARGIN 6
MENU IMMEDIATE
# http://www.colorpicker.com/
MENU color border 0 #00000000 #00000000 none
MENU color title 0 #FFF5B800 #00000000 std
MENU color sel 7;37;40 #FF000000 #FFFFFFFF all
MENU color hotsel 1;7;37;40 #FFFF0000 #FFC0C0C0 all
MENU color hotkey 1;7;37;40 #FF0000CC #FFC0C0C0 all
MENU color tabmsg 1;31;40 #FF808080 #00000000 std
MENU color help 1;31;40 #FFFFFFFF #FF000000 none
MENU color timeout_msg 0 #FFFFB300 #00000000 none
MENU color timeout 0 #FFFF0000 #FF000000 none
MENU color cmdline 0 #FFFFFFFF #FF000000 none
MENU color cmdmark 1;36;40 #C000FFFF #FF000000 std
MENU color scrollbar 30;44 #FF00FF00 #FF000000 std
MENU color msg07 0 #FF000000 #00FFFFFF none
MENU BACKGROUND /{{ bdisk['uxname'] }}.png

LABEL check
MENU LABEL Your best supported kernel should be detected automatically.
COM32 ifcpu64.c32
APPEND {{ bdisk['uxname'] }}_64 -- {{ bdisk['uxname'] }}_32
MENU DEFAULT



LABEL local_override
MENU LABEL Local ^Boot
localboot 0
TEXT HELP
Boot from the local system instead.
ENDTEXT

LABEL reboot
MENU LABEL ^Reboot
COM32 reboot.c32
TEXT HELP
Reboot the machine
ENDTEXT

MENU SEPARATOR

## 64 BIT
MENU BEGIN 64BIT
MENU LABEL ^1) 64-Bit ...
ONTIMEOUT {{ bdisk['uxname'] }}_64
LABEL {{ bdisk['uxname'] }}_ram_64
MENU LABEL ^1) {{ bdisk['pname'] }} (run from RAM)
LINUX /boot/{{ bdisk['uxname'] }}.64.kern
INITRD /boot/{{ bdisk['uxname'] }}.64.img
APPEND copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
TEXT HELP
64-bit, run from RAM
ENDTEXT
MENU DEFAULT

LABEL {{ bdisk['uxname'] }}_64
MENU LABEL ^1) {{ bdisk['pname'] }} (Default)
LINUX /boot/{{ bdisk['uxname'] }}.64.kern
INITRD /boot/{{ bdisk['uxname'] }}.64.img
APPEND archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
TEXT HELP
Same as the above, except run directly from the CD-
don't copy the image to RAM. (Best for lower-memory boxes)
ENDTEXT

MENU END

MENU BEGIN 32BIT
MENU LABEL ^2) 32-Bit ...
ONTIMEOUT {{ bdisk['uxname'] }}_32

## 32 BIT
LABEL {{ bdisk['uxname'] }}_ram_32
MENU LABEL ^1) {{ bdisk['pname'] }} (run from RAM)
LINUX /boot/{{ bdisk['uxname'] }}.32.kern
INITRD /boot/{{ bdisk['uxname'] }}.32.img
APPEND copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
TEXT HELP
32-bit, run from RAM
ENDTEXT
MENU DEFAULT

LABEL {{ bdisk['uxname'] }}_32
MENU LABEL ^2) {{ bdisk['pname'] }} (Default)
LINUX /boot/{{ bdisk['uxname'] }}.32.kern
INITRD /boot/{{ bdisk['uxname'] }}.32.img
APPEND archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
TEXT HELP
Same as the above, except run directly from the CD-
don't copy the image to RAM. (Best for lower-memory boxes)
ENDTEXT

MENU END

4
extra/templates/EFI/base.conf.j2 Normal file
View File

@ -0,0 +1,4 @@
title {{ bdisk['pname'] }} (Media)
linux /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.{% if efi is defined %}efi{% else %}kern{% endif %}
initrd /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.img
options archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}

2
extra/templates/EFI/loader.conf.j2 Normal file
View File

@ -0,0 +1,2 @@
timeout 3
default {{ bdisk['uxname'] }}

4
extra/templates/EFI/ram.conf.j2 Normal file
View File

@ -0,0 +1,4 @@
title {{ bdisk['pname'] }} (RAM)
linux /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.{% if efi is defined %}efi{% else %}kern{% endif %}
initrd /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.img
options copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}

2
extra/templates/EFI/uefi1.conf.j2 Normal file
View File

@ -0,0 +1,2 @@
title UEFI Shell (v1)
efi /EFI/shellx64_v1.efi

2
extra/templates/EFI/uefi2.conf.j2 Normal file
View File

@ -0,0 +1,2 @@
title UEFI Shell (v2)
efi /EFI/shellx64_v2.efi

1
extra/templates/VERSION_INFO.txt.j2
View File

@ -3,3 +3,4 @@ Build: {{ build['name'] }}
Time: {{ build['time'] }}
Machine: {{ hostname }}
User: {{ build['user'] }}{% if build['realuser'] is defined and build['realuser'] > 0 %} ({{ build['realuser'] }}){% endif %}


8
extra/templates/iPXE/EMBED.j2 Normal file
View File

@ -0,0 +1,8 @@
#!ipxe

dhcp
## TODO: signed kernel and initrd
#imgtrust --permanent
#imgverify vmlinuz path/to/vmlinuz.sig
#imgverify initrd path/to/initrd.sig
chain {{ ipxe['uri'] }}

1
overlay/etc/systemd/system/getty.target.wants/getty@tty1.service
View File

@ -1 +0,0 @@
/usr/lib/systemd/system/getty@.service

0
overlay/32/.keepme → overlay/i686/.keepme
View File

0
overlay/64/.keepme → overlay/x86_64/.keepme
View File