safety checkin... getting super close

.gitignore

@@ -25,6 +25,7 @@
 /logs
 *.swp
 *.lck
+*~
 /extrasrc
 
 # You should really generate local copies of these, as they're pretty private.

TODO

@@ -1,2 +0,0 @@
-- templating via jinja2:
--- for syslinux/isolinux configs

bdisk/bchroot.py

@@ -29,35 +29,35 @@ def chroot(chrootdir, chroot_hostname, cmd = '/root/pre-build.sh'):
         mounts.append(m.mountpoint)
     # mount the chrootdir... onto itself. as a bind mount. it's so stupid, i know. see https://bugs.archlinux.org/task/46169
     if chrootdir not in mounts:
-        subprocess.call(['mount', '--bind', chrootdir, chrootdir])
+        subprocess.call(['/bin/mount', '--bind', chrootdir, chrootdir])
 ### The following mountpoints don't seem to mount properly with pychroot. save it for v3.n+1. TODO. ###
     # bind-mount so we can resolve things inside
     if (chrootdir + '/etc/resolv.conf') not in mounts:
-        subprocess.call(['mount', '--bind', '-o', 'ro', '/etc/resolv.conf', chrootdir + '/etc/resolv.conf'])
+        subprocess.call(['/bin/mount', '--bind', '-o', 'ro', '/etc/resolv.conf', chrootdir + '/etc/resolv.conf'])
     # mount -t proc to chrootdir + '/proc' here
     if (chrootdir + '/proc') not in mounts:
-        subprocess.call(['mount', '-t', 'proc', '-o', 'nosuid,noexec,nodev', 'proc', chrootdir + '/proc'])
+        subprocess.call(['/bin/mount', '-t', 'proc', '-o', 'nosuid,noexec,nodev', 'proc', chrootdir + '/proc'])
     # rbind mount /sys to chrootdir + '/sys' here
     if (chrootdir + '/sys') not in mounts:
-        subprocess.call(['mount', '-t', 'sysfs', '-o', 'nosuid,noexec,nodev,ro', 'sys', chrootdir + '/sys'])
+        subprocess.call(['/bin/mount', '-t', 'sysfs', '-o', 'nosuid,noexec,nodev,ro', 'sys', chrootdir + '/sys'])
     # mount the efivars in the chroot if it exists on the host. i mean, why not?
     if '/sys/firmware/efi/efivars' in mounts:
         if (chrootdir + '/sys/firmware/efi/efivars') not in mounts:
-            subprocess.call(['mount', '-t', 'efivarfs', '-o', 'nosuid,noexec,nodev', 'efivarfs', chrootdir + '/sys/firmware/efi/efivars'])
+            subprocess.call(['/bin/mount', '-t', 'efivarfs', '-o', 'nosuid,noexec,nodev', 'efivarfs', chrootdir + '/sys/firmware/efi/efivars'])
     # rbind mount /dev to chrootdir + '/dev' here
     if (chrootdir + '/dev') not in mounts:
-        subprocess.call(['mount', '-t', 'devtmpfs', '-o', 'mode=0755,nosuid', 'udev', chrootdir + '/dev'])
+        subprocess.call(['/bin/mount', '-t', 'devtmpfs', '-o', 'mode=0755,nosuid', 'udev', chrootdir + '/dev'])
     if (chrootdir + '/dev/pts') not in mounts:
-        subprocess.call(['mount', '-t', 'devpts', '-o', 'mode=0620,gid=5,nosuid,noexec', 'devpts', chrootdir + '/dev/pts'])
+        subprocess.call(['/bin/mount', '-t', 'devpts', '-o', 'mode=0620,gid=5,nosuid,noexec', 'devpts', chrootdir + '/dev/pts'])
     if '/dev/shm' in mounts:
         if (chrootdir + '/dev/shm') not in mounts:
-            subprocess.call(['mount', '-t', 'tmpfs', '-o', 'mode=1777,nosuid,nodev', 'shm', chrootdir + '/dev/shm'])
+            subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'mode=1777,nosuid,nodev', 'shm', chrootdir + '/dev/shm'])
     if '/run' in mounts:
         if (chrootdir + '/run') not in mounts:
-            subprocess.call(['mount', '-t', 'tmpfs', '-o', 'nosuid,nodev,mode=0755', 'run', chrootdir + '/run'])
+            subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'nosuid,nodev,mode=0755', 'run', chrootdir + '/run'])
     if '/tmp' in mounts:
         if (chrootdir + '/tmp') not in mounts:
-            subprocess.call(['mount', '-t', 'tmpfs', '-o', 'mode=1777,strictatime,nodev,nosuid', 'tmp', chrootdir + '/tmp'])
+            subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'mode=1777,strictatime,nodev,nosuid', 'tmp', chrootdir + '/tmp'])
 
     print("Now performing '{0}' in chroot for {1}...".format(cmd, chrootdir))
     print("You can view the progress via:\n\n\ttail -f {0}/var/log/chroot_install.log\n".format(chrootdir))
@@ -70,5 +70,4 @@ def chroot(chrootdir, chroot_hostname, cmd = '/root/pre-build.sh'):
     return(chrootdir)
 
 def chrootUnmount(chrootdir):
-    # TODO: https://github.com/pkgcore/pychroot/issues/22 try to do this more pythonically. then we can remove subprocess
     subprocess.call(['umount', '-lR', chrootdir])

bdisk/bdisk.py

@@ -5,21 +5,18 @@ import bchroot
 import build
 
 # we need to:
-# 8.) build.chrootClean (TODO) see jenny_craig in old bdisk. i can *probably* do this within the chroot for the most part as part of pre-build.sh
 # 9.) build.genImg (TODO)- build the squashed image, etc. see will_it_blend in old bdisk
-# 10.) build.genUEFI (TODO)- build the uefi binary/bootloading. see stuffy in old bdisk
+# 9.5) copy the files also in the same script. after the commented-out mtree-generation
-# 11.) build.genISO (TODO)- build the .iso file (full boot). see yo_dj in old bdisk
 #
 # we also need to figure out how to implement "mentos" (old bdisk) like functionality, letting us reuse an existing chroot install if possible to save time for future builds.
 #   if not, though, it's no big deal.
 if __name__ == '__main__':
-    # TODO: config for chrootdir, dlpath
     conf = host.parseConfig(host.getConfig())[1]
     prep.dirChk(conf)
     prep.buildChroot(conf['build'])
-    prep.prepChroot(conf['build']['basedir'] + '/extra/templates', conf['build'], conf['bdisk'])
+    prep.prepChroot(conf['build'], conf['bdisk'])
     arch = conf['build']['arch']
     for a in arch:
         bchroot.chroot(conf['build']['chrootdir'] + '/root.' + a, 'bdisk.square-r00t.net')
         bchroot.chrootUnmount(conf['build']['chrootdir'] + '/root.' + a)
-    #build.chrootClean(conf['build'])
+    prep.postChroot(conf['build'])

bdisk/build.py

@@ -2,109 +2,271 @@ import os
 import tarfile
 import shutil
 import glob
+import subprocess
+import hashlib
+import jinja2
+from urllib.request import urlopen
 
 
-def chrootClean(build):
+def genImg(build):
-    chrootdir = build['chrootdir']
     arch = build['arch']
-    backupdir = build['dlpath'] + '/' + 'bak'
+    chrootdir = build['chrootdir']
-    os.makedirs(backupdir, exist_ok = True)
+    archboot = build['archboot']
-    ## Save some stuff that needs to be retained.
+    hashes = {}
-    # Compress the pacman cache.
+    hashes['sha256'] = {}
+    hashes['md5'] = {}
     for a in arch:
-        os.makedirs(chrootdir + '/root.' + a + '/usr/local/pacman', exist_ok = True)
+        # Create the squashfs image
-        tarball = chrootdir + '/root.' + a + '/usr/local/pacman/pacman.db.tar.xz'
+        airoot = archboot + '/' + a + '/'
-        dbdir = chrootdir + '/root.' + a + '/var/lib/pacman/local'
+        squashimg = airoot + 'airootfs.sfs'
-        print("Now cleaning {0}/root.{1}. Please wait...".format(chrootdir, a))
+        os.makedirs(airoot, exist_ok = True)
-        if os.path.isfile(tarball):
+        print("Generating squashed filesystem image for {0}. Please wait...".format(chrootdir))
-            os.remove(tarball)
+        cmd = ['/usr/bin/squashfs-tools', chrootdir, squashimg, '-noappend', '-comp', 'xz']
-        with tarfile.open(name = tarball, mode = 'w:xz') as tar:  # if this balks, try x:xz
+        proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, bufsize = 1)
-            tar.add(dbdir, arcname = os.path.basename(dbdir))
+        #for line in iter(proc.stdout.readline, b''):
-        # Cut out the fat
+        for line in iter(proc.stdout.readline, ''):
-        # The following are intended as "overrides" of the paths we'll be deleting.
+            print(line)
-        backup = ['/var/lib/pacman/local',
+        p.stdout.close()
-                    '/usr/share/locale/locale.alias',
+        p.wait()
-                    '/usr/share/zoneinfo/EST5EDT',
+        # Generate the checksum files
-                    '/usr/share/zoneinfo/UTC',
+        print("Now generating SHA256 and MD5 hash checksum files for {0}. Please wait...".format(squashimg))
-                    '/usr/share/locale/en',
+        hashes['sha256'][a] = hashlib.sha256()
-                    '/usr/share/locale/en_US',
+        hashes['md5'][a] = hashlib.md5()
-                    '/usr/share/locale/en_GB']
+        with open(squashimg, 'rb') as f:
-        # And these are what we remove.
+            while True:
-        delete = ['/usr/share/locale/',
+                stream = f.read(65536)  # 64kb chunks
-                    '/var/cache/pacman/',
+                if not stream:
-                    '/var/cache/pkgfile/',
+                    break
-                    '/var/cache/apacman/pkg/',
+                # NOTE: these items are hashlib objects, NOT strings!
-                    '/var/lib/pacman/',
+                hashes['sha256'][a].update(stream)
-                    '/var/abs/local/yaourtbuild/',
+                hashes['md5'][a].update(stream)
-                    '/usr/share/zoneinfo',
+        with open(airoot + 'airootfs.sha256', 'w+') as f:
-                    '/root/.gnupg',
+            f.write("{0}  airootfs.sfs".format(hashes['sha256'][a].hexdigest()))
-                    '/tmp/',
+        with open(airoot + 'airootfs.md5', 'w+') as f:
-                    '/var/tmp/',
+            f.write("{0}  airootfs.sfs".format(hashes['md5'][a].hexdigest()))
-                    '/var/abs/',
+
-                    '/run/',
+
-                    '/boot/',
+def genUEFI(build, bdisk):
-                    '/usr/src/',
+    arch = build['arch']
-                    '/var/log/',
+    # 32-bit EFI implementations are nigh nonexistant.
-                    '/.git']
+    # We don't really need to worry about them.
-        delete['files'] = ['/root/.bash_history',
+    # Plus there's always multiarch.
-                        '/root/apacman*',
+    # I can probably do this better with a dict... TODO.
-                        '/root/iso.pkgs*',
+    if 'x86_64' in arch:
-                        '/root/packages.*',
+        os.makedirs(tempdir + '/EFI/boot', exist_ok = True)
-                        '/root/pre-build.sh',
+        tempdir = build['tempdir']
-                        '/root/.viminfo',
+        basedir = build['basedir']
-                        '/root/.bashrc']
+        templates_dir = build['basedir'] + '/extra/templates'
-        # First we backup files. We don't need to create backup['dirs']
+        efiboot_img = tempdir + '/EFI/' + bdisk['name'] + '/efiboot.img'
-        # since they should be empty. If not, they go in backup['files'].
+        ## Download the EFI shells if we don't have them.
-        for f in backup['files']:
+        # For UEFI 2.3+ (http://sourceforge.net/apps/mediawiki/tianocore/index.php?title=UEFI_Shell)
-            #os.makedirs(backupdir + '/root.' + a + os.path.dirname(f), exist_ok = True)
+        if not os.path.isfile(tempdir + '/EFI/shellx64_v2.efi'):
-            #shutil.copy2(chrootdir + '/root.' + a + f, backupdir + '/root.' + a + f)
+            shell2_path = tempdir + '/EFI/shellx64_v2.efi'
-            for root, dirs, files in os.walk(f):
+            print("You are missing {0}. We'll download it for you.".format(shell2_path))
-                for item in files:
+            shell2_url = 'https://github.com/tianocore/edk2/blob/master/ShellBinPkg/UefiShell/X64/Shell.efi?raw=true'
-                    src_path = os.path.join(root, item)
+            shell2_fetch = urlopen(shell2_url)
-                    dst_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
+            with open(shell2_path, 'wb+') as dl:
-                    if os.path.exists(dst_path):
+                dl.write(shell2_fetch.read())
-                        if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime:
+            shell2_fetch.close()
-                            shutil.copy2(src_path, dst_path)
+        # Shell for older versions (http://sourceforge.net/apps/mediawiki/tianocore/index.php?title=Efi-shell)
+        # TODO: is there an Arch package for this? can we just install that in the chroot and copy the shell binaries?
+        if not os.path.isfile(tempdir + '/EFI/shellx64_v1.efi'):
+            shell1_path = tempdir + '/EFI/shellx64_v1.efi'
+            print("You are missing {0}. We'll download it for you.".format(shell1_path))
+            shell1_url = 'https://github.com/tianocore/edk2/blob/master/EdkShellBinPkg/FullShell/X64/Shell_Full.efi?raw=true'
+            shell1_fetch = urlopen(shell1_url)
+            with open(shell1_path, 'wb+') as dl:
+                dl.write(shell1_fetch.read())
+            shell1_fetch.close()
+        print("Now configuring UEFI bootloading...")
+        ## But wait! That's not all! We need more binaries.
+        # http://blog.hansenpartnership.com/linux-foundation-secure-boot-system-released/
+        shim_url = 'http://blog.hansenpartnership.com/wp-uploads/2013/'
+        for f in ('bootx64.efi', 'HashTool.efi'):
+            if not os.path.isfile(tempdir + '/EFI/boot/' + f):
+                url = shim_url + f
+                url_fetch = urlopen(url)
+                with open(tempdir + '/EFI/boot/' + f) as dl:
+                    dl.write(url_fetch.read())
+                url_fetch.close()
+        # And we also need the systemd efi bootloader.
+        if os.path.isfile(tempdir + '/EFI/boot/loader.efi'):
+            os.remove(tempdir + '/EFI/boot/loader.efi')
+        shutil.copy2(chrootdir + '/root.x86_64/usr/lib/systemd/boot/efi/systemd-bootx64.efi', tempdir + '/EFI/boot/loader.efi')
+        # And the accompanying configs for the systemd efi bootloader, too.
+        loader = jinja2.FileSystemLoader(templates_dir)
+        env = jinja2.Environment(loader = loader)
+        os.makedirs(tempdir + '/loader/entries', exist_ok = True)
+        for t in ('loader', 'ram', 'base', 'uefi2', 'uefi1'):
+            if t == 'base':
+                fname = bdisk['uxname'] + '.conf'
+            elif ('uefi1', 'uefi2') in t:
+                fname = t + '.conf'
             else:
-                        shutil.copy2(src_path, dst_path)
+                fname = bdisk['uxname'] + '_' + t + '.conf'
-                for item in dirs:
+            if t == 'loader':
-                    src_path = os.path.join(root, item)
+                tplpath = tempdir + '/loader/'
-                    dst_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
+                fname = 'loader.conf'  # we change the var from above because it's an oddball.
-                    os.makedirs(dst_path, exist_ok = True)
-        # Now we delete the above.
-        for f in delete['files']:
-            for x in glob.glob(chrootdir + '/root.' + a + f):
-                os.remove(x)
-        for d in delete['dirs']:
-            for x in glob.glob(chrootdir + '/root.' + a + d):
-                #os.remove(x)
-                shutil.rmtree(x)
-        # And restore the dirs/files
-        for d in backup['dirs']:
-            os.makedirs(chrootdir + '/root.' + a + d, exist_ok = True)
-        for f in backup['files']:
-            #os.makedirs(chrootdir + '/root.' + a + os.path.dirname(f), exist_ok = True)
-            #shutil.copy2(backupdir + '/root.' + a + f, chrootdir + '/root.' + a + f)
-            for root, dirs, files in os.walk(f):
-                for item in files:
-                    src_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
-                    dst_path = os.path.join(root, item)
-                    if os.path.exists(dst_path):
-                        if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime:
-                            shutil.copy2(src_path, dst_path)
             else:
-                        shutil.copy2(src_path, dst_path)
+                tplpath = tempdir + '/loader/entries/'
-                for item in dirs:
+            tpl = env.get_template(t)
-                    src_path = os.path.join(backupdir + '/root.' + a, src_path.replace(f, ''))
+            tpl_out = tpl.render(build = build, bdisk = bdisk)
-                    dst_path = os.path.join(root, item)
+            with open(tplpath + fname, "w+") as f:
-                    os.makedirs(dst_path, exist_ok = True)
+                f.write(tpl_out)
-        #shutil.rmtree(backupdir)
+        # And we need to get filesizes (in bytes) for everything we need to include in the ESP.
+        # This is more important than it looks.
+        sizetotal = 786432  # we start with 768KB and add to it for wiggle room
+        sizefiles = ['/boot/' + bdisk['uxname'] + '.64.img',
+                    '/EFI/boot/bootx64.efi',
+                    '/EFI/boot/loader.efi',
+                    '/EFI/boot/HashTool.efi',
+                    '/EFI/shellx64_v1.efi',
+                    '/EFI/shellx64_v2.efi']
+        for i in sizefiles:
+            sizetotal += os.path.getsize(tempdir + i)
+        # Loader configs
+        for (path, dirs, files) in os.walk(tempdir + '/loader/'):
+            for file in files:
+                fname = os.path.join(path, file)
+                sizetotal += os.path.getsize(fname)
+        # And now we create the file...
+        print("Now creating a {0} bytes EFI ESP image at {1}. Please wait...".format(sizetotal, efiboot_img))
+        if os.path.isfile(efiboot_img):
+            os.remove(efiboot_img)
+        with open(efiboot_img, 'w+') as f:
+            f.truncate(sizetotal)
+        cmd = ['/sbin/mkfs.vfat', '-F', '32', '-n', bdisk['name'] + '_EFI', efiboot_img]
+        subprocess.call(cmd)
+        cmd = ['/bin/mount', efiboot_img, build['mountpt']]
+        subprocess.call(cmd)
+        os.makedirs(build['mountpt'] + '/' + bdisk['name'])
+        os.makedirs(build['mountpt'] + '/EFI/boot')
+        os.makedirs(build['mountpt'] + '/loader/entries')
+        # Ready for some deja vu? This is because it uses an embedded version as well for hybrid ISO.
+        # I think.
+        # TODO: just move this to a function instead, with "efi" as a param and change
+        # the templates to use "if efi == 'yes'" instead.
+        # function should set the "installation" path for the conf as well based on the value of efi
+        # parameter.
+        loader = jinja2.FileSystemLoader(templates_dir)
+        env = jinja2.Environment(loader = loader)
+        os.makedirs(build['mountpt'] + 'loader/entries', exist_ok = True)
+        for t in ('loader', 'ram', 'base', 'uefi2', 'uefi1'):
+            if t == 'base':
+                fname = bdisk['uxname'] + '.conf'
+            elif ('uefi1', 'uefi2') in t:
+                fname = t + '.conf'
+            else:
+                fname = bdisk['uxname'] + '_' + t + '.conf'
+            if t == 'loader':
+                tplpath = build['mountpt'] + '/loader/'
+                fname = 'loader.conf'  # we change the var from above because it's an oddball.
+            else:
+                tplpath = build['mountpt'] + '/loader/entries/'
+            tpl = env.get_template(t)
+            tpl_out = tpl.render(build = build, bdisk = bdisk, efi = 'yes')
+            with open(tplpath + fname, "w+") as f:
+                f.write(tpl_out)
+            for x in ('bootx64.efi', 'HashTool.efi', 'loader.efi', 'shellx64_v1.efi', 'shellx64_v2.efi'):
+                y = tempdir + '/EFI/boot/' + x
+                z = mountpt + '/EFI/boot/' + x
+                if os.path.isfile(z):
+                    os.remove(z)
+                shutil.copy(y, z)
+        cmd = ['/bin/umount', mountpt]
+        subprocess.call(cmd)
+        return(efiboot_img)
 
-def genImg():
+def genISO(conf):
-    pass
+    build = conf['build']
+    bdisk = conf['bdisk']
+    archboot = build['archboot']
+    tempdir = build['tempdir']
+    builddir = tempdir + '/' + bdisk['name']
+    extradir = build['basedir'] + '/extra/'
+    # arch[0] is safe to use, even if multiarch, because the only cases when it'd be ambiguous
+    # is when x86_64 is specifically set to [0]. See host.py's parseConfig().
+    syslinuxdir = build['chrootdir'] + '/root.' + arch[0] + '/usr/lib/syslinux/'
+    sysl_tmp = tempdir + '/isolinux/'
+    ver = build['ver']
+    isofile = '{0}-{1}.iso'.format(bdisk['uxname'], bdisk['ver'])
+    isopath = build['isodir'] + '/' + isofile
+    arch = build['arch']
+    # In case we're building a single-arch ISO...
+    if len(arch) == 1:
+        isolinux_cfg = extradir + 'templates/BIOS/isolinux.cfg.arch.j2'
+        if arch[0] == 'i686':
+            bitness = '32'
+        elif arch[0] == 'x86_64':
+            bitness = '64'
+    if build['ipxe']:
+        ipxe = conf['ipxe']
+        if ipxe['iso']:
+            minifile = '{0}-{1}-mini.iso'.format(bdisk['uxname'], bdisk['ver'])
+            minipath = build['isodir'] + '/' + minifile
+        if ipxe['usb']:
+            usbfile = '{0}-{1}-mini.usb.img'.format(bdisk['uxname'], bdisk['ver'])
+            minipath = build['isodir'] + '/' + usbfile
+    else:
+        isolinux_cfg = extradir + 'templates/BIOS/isolinux.cfg.multi.j2'
+        bitness = False
+    if os.path.isfile(isopath):
+        os.remove(isopath)
+    if archboot != tempdir + '/' + bdisk['name']:  # best to use static concat here... 
+        if os.path.isdir(builddir):
+            shutil.rmtree(builddir, ignore_errors = True)
+        shutil.copytree(archboot, builddir)
+    # Copy isolinux files
+    print("Now staging some files for ISO preparation. Please wait...")
+    isolinux_files = ['isolinux.bin',
+                    'vesamenu.c32',
+                    'linux.c32',
+                    'reboot.c32']
+    # TODO: implement debugging mode in bdisk
+    #if debug:
+    #   isolinux_files[0] = 'isolinux-debug.bin'
+    os.makedirs(sysl_tmp, exist_ok = True)
+    for f in isolinux_files:
+        if os.path.isfile(sysl_tmp + f):
+            os.remove(sysl_tmp + f)
+        shutil.copy2(syslinuxdir + f, sysl_tmp + f)
+    ifisolinux_files = ['ldlinux.c32',
+                        'libcom32.c32',
+                        'libutil.c32',
+                        'ifcpu64.c32']
+    for f in ifisolinux_files:
+        if os.path.isfile(sysl_tmp + f):
+            os.remove(sysl_tmp + f)
+        shutil.copy2(syslinuxdir + f, sysl_tmp + f)
+    # And we need to build the ISO!
+    print("Now generating the full ISO at {0}. Please wait.".format(isopath))
+    cmd = ['/usr/bin/xorriso',
+        '-as', 'mkisofs',
+        '-iso-level', '3',
+        '-full-iso9660-filenames',
+        '-volid', bdisk['name'],
+        '-appid', bdisk['desc'],
+        '-publisher', bdisk['dev'],
+        '-preparer', 'prepared by ' + bdisk['dev'],
+        '-eltorito-boot', 'isolinux/isolinux.bin',
+        '-eltorito-catalog', 'isolinux/boot.cat',
+        '-no-emul-boot',
+        '-boot-load-size', '4',
+        '-boot-info-table',
+        '-isohybrid-mbr', syslinuxdir + 'isohdpfx.bin',
+        '-eltorito-alt-boot',
+        '-e', 'EFI/' + bdisk['name'] + '/efiboot.img',
+        '-no-emul-boot',
+        '-isohybrid-gpt-basdat',
+        '-output', isopath,
+        tempdir]
+    proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, bufsize = 1)
+    #for line in iter(proc.stdout.readline, b''):
+    for line in iter(proc.stdout.readline, ''):
+        print(line)
+    p.stdout.close()
+    p.wait()
 
-def genUEFI():
+def cleanUp():
-    pass
+    # TODO: clear out all of tempdir?
-
-def genISO():
     pass

bdisk/ipxe.py

@@ -0,0 +1,13 @@
+import os
+import shutil
+import jinja2
+import gitpython
+
+
+def buildIPXE(conf):
+    build = conf['build']
+    bdisk = conf['bdisk']
+    ipxe = conf['ipxe']
+    templates_dir = build['basedir'] + '/extra/templates'
+    patches_dir = build['basedir'] + 
+    pass

bdisk/prep.py

@@ -12,6 +12,7 @@ import datetime
 from urllib.request import urlopen
 import host  # bdisk.host
 
+
 def dirChk(config_dict):
     # Make dirs if they don't exist
     for d in ('archboot', 'isodir', 'mountpt', 'srcdir', 'tempdir'):
@@ -143,20 +144,21 @@ def buildChroot(build):
             os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0)
         # and copy over the files. again, chown to root.
         for file in prebuild_overlay['files']:
-            shutil.copy2(extradir + '/pre-build.d/' + file, chrootdir + '/root.' + a + '/' + file)
+            shutil.copy2(extradir + '/pre-build.d/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
             os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0)
         # do the same for arch-specific stuff.
         for dir in prebuild_arch_overlay[a]['dirs']:
             os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
             os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0)
         for file in prebuild_arch_overlay[a]['files']:
-            shutil.copy2(extradir + '/pre-build.d/' + a + '/' + file, chrootdir + '/root.' + a + '/' + file)
+            shutil.copy2(extradir + '/pre-build.d/' + a + '/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
             os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0)
 
-def prepChroot(templates_dir, build, bdisk):
+def prepChroot(build, bdisk):
     chrootdir = build['chrootdir']
     arch = build['arch']
     bdisk_repo_dir = build['basedir']
+    templates_dir = bdisk_repo_dir + '/extra/templates'
     build = {}
     # let's prep some variables to write out the version info.txt
     # get the git tag and short commit hash
@@ -174,8 +176,50 @@ def prepChroot(templates_dir, build, bdisk):
     loader = jinja2.FileSystemLoader(templates_dir)
     env = jinja2.Environment(loader = loader)
     tpl = env.get_template('VERSION_INFO.txt.j2')
-    tpl_out = tpl.render(build = build, hostname = hostname)
+    tpl_out = tpl.render(build = build, hostname = host.getHostname())
     for a in arch:
         with open(chrootdir + '/root.' + a + '/root/VERSION_INFO.txt', "w+") as f:
             f.write(tpl_out)
     return(build)
+
+def postChroot(build):
+    dlpath = build['dlpath']
+    chrootdir = build['chrootdir']
+    arch = build['arch']
+    overdir = build['basedir'] + '/overlay/'
+    postbuild_overlay = {}
+    postbuild_arch_overlay = {}
+    for x in arch:
+        postbuild_arch_overlay[x] = {}
+        for y in ['files', 'dirs']:
+            postbuild_overlay[y] = []
+            postbuild_arch_overlay[x][y] = []
+    for path, dirs, files in os.walk(overdir):
+        postbuild_overlay['dirs'].append(path + '/')
+        for file in files:
+            postbuild_overlay['files'].append(os.path.join(path, file))
+    for x in postbuild_overlay.keys():
+        postbuild_overlay[x][:] = [re.sub('^' + overdir, '', s) for s in postbuild_overlay[x]]
+        postbuild_overlay[x] = list(filter(None, postbuild_overlay[x]))
+        for y in postbuild_arch_overlay.keys():
+            postbuild_arch_overlay[y][x][:] = [i for i in postbuild_overlay[x] if i.startswith(y)]
+            postbuild_arch_overlay[y][x][:] = [re.sub('^' + y + '/', '', s) for s in postbuild_arch_overlay[y][x]]
+            postbuild_arch_overlay[y][x] = list(filter(None, postbuild_arch_overlay[y][x]))
+        postbuild_overlay[x][:] = [y for y in postbuild_overlay[x] if not y.startswith(('x86_64','i686'))]
+    postbuild_overlay['dirs'].remove('/')
+    # create the dir structure. these should almost definitely be owned by root.
+    for a in arch:
+        for dir in postbuild_overlay['dirs']:
+            os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
+            os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0, follow_symlinks = False)
+        # and copy over the files. again, chown to root.
+        for file in postbuild_overlay['files']:
+            shutil.copy2(overdir + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
+            os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0, follow_symlinks = False)
+        # do the same for arch-specific stuff.
+        for dir in postbuild_arch_overlay[a]['dirs']:
+            os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
+            os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0, follow_symlinks = False)
+        for file in postbuild_arch_overlay[a]['files']:
+            shutil.copy2(overdir + a + '/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False)
+            os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0, follow_symlinks = False)

docs/CREDITS

@@ -0,0 +1,12 @@
+iPXE:
+Thanks to "eworm" for his work on the AUR iPXE-git package:
+https://aur.archlinux.org/packages/ipxe-git/
+
+and specifically the following patches:
+http://www.eworm.de/download/linux/ipxe-0001-git-version.patch
+http://www.eworm.de/download/linux/ipxe-0002-banner.patch
+http://www.eworm.de/download/linux/ipxe-0003-iso-efi.patch
+http://www.eworm.de/download/linux/ipxe-0004-fix-no-pie-workaround.patch
+
+
+thanks to jthan, even though he drives me batty sometimes.

docs/THANKS

@@ -0,0 +1 @@
+CREDITS

docs/TODO

@@ -1,3 +1,8 @@
+## Missing v2.x functionality ##
+-i_am_a_racecar optimizations
+-see also: grep -HnR bdisk/*.py
+-TFTP, HTTP, RSYNC, git
+
 ## General ##
 
 -include benchmarking
@@ -5,8 +10,7 @@
 -- https://code.google.com/p/byte-unixbench/
 -- https://github.com/akopytov/sysbench
 -- (http://blog.due.io/2014/linode-digitalocean-and-vultr-comparison/ etc.)
--package in AUR
+-implement pyalpm to decreate dependency on chroot pacman-ing?
--base rewrite in python. pyalpm may come in handy here.
 
 
 ## NETWORKING ##
@@ -23,28 +27,27 @@
 
 ## Building ##
 
--GUMMIBOOT IS GONE FROM THE REPOS. I could repackage it, but better to just see what the hell archiso's doing.
 -WISH: Better logging[0]
--WISH: signing for secureboot releases (PreLoader and gummiboot handle this okay, but require manual intervention
+-WISH: signing for secureboot releases (PreLoader and loader.efi handle this okay, but require manual intervention)
--use manual chrooting functions ONLY if distro not detected as arch. if /usr/bin/systemd-nspawn exists, use that instead
+-does loader.efi support splash backgrounds? can i implement that differently somehow?
---does arch-chroot work across all distros? see https://wiki.archlinux.org/index.php/Install_bundled_32-bit_system_in_Arch64 and https://wiki.archlinux.org/index.php/Chroot
---i think this might be unnecessary. testing across other major distros is necessary, but i think i can just use the chroot'd arch-chroot
--tweak build.conf (and build.conf.sample) to source the pwd and set as BASEDIR ***if*** the project resources are present in pwd, otherwise throw warning
---this is half-done;PWD is currently used by default.
--does gummiboot? loader? wtfever it's called support splash backgrounds? can i implement that differently somehow?
 --yes, see e.g. https://www.reddit.com/r/archlinux/comments/3bwgf0/where_put_the_splasharchbmp_to_splash_screen_boot/
 -strip out/remove unnecessary and orphan packages (e.g. gcc, make, automake, etc.)
 -incorporate iPXE tweaks:
 --http://ipxe.org/crypto 
 --http://ipxe.org/cmd/imgtrust
 --http://ipxe.org/cmd/imgverify
---enable use of custom CA/self-signed certs for HTTPS etc. DONE, partially. need to incorporate codesign certs/keys. routines, conf variables
+--enable use of custom CA/self-signed certs for HTTPS etc.
+--signed kernel and initrd for ipxe:
+---#imgtrust --permanent
+---#imgverify vmlinuz path/to/vmlinuz.sig
+---#imgverify initrd path/to/initrd.sig
+---DONE, partially. need to incorporate codesign certs/keys. routines, conf variables
 -enable mirror= kernel commandline.
 --if mirror_(NAME) is present, use that as repo name.
 --if it starts with /, treat as mirrorlist (Include); otherwise use Server = 
 --if it has mirror_SIG-X, set signature options e.g. _SIG-N would be "SigLevel = Never"
 -iPXE background support. sed -rf "${BASEDIR}/src/ipxe_local/script.sed" ${SRCDIR}/ipxe/src/config/general.h ; sed -rf "${BASEDIR}/src/ipxe_local/script2.sed" ${SRCDIR}/ipxe/src/config/console.h
---note that iPXE VESAFB console is not (yet) supported in EFI, so this is on hold.
+--note that iPXE VESAFB console is not (yet) supported in EFI, so this is on hold. check into this to see if it has changed.
 
 ## Split into Separate Tools CD ##
 
@@ -57,12 +60,4 @@ __________________________________________________________
 FOOTNOTES:
 
 
-[0] I'd really like to implement the following in build.conf; like:
+[0] Debugging mode would probably suffice for logging? Maybe? There are plenty of pythonic ways to do logging too.
-http://forums.fedoraforum.org/showthread.php?t=275743
-# The following is the setting for "verbosity". A more accurate way of saying it is how output should be handled.
-# Note that for it to be properly parsed, it MUST be in the form of a linear array (e.g. VAR=(1 2 3) ).
-# '| tee -a ${BASEDIR}/logs/${FUNCNAME}.$(date +%s)' means "display output for STDOUT and STDERR, and also log STDOUT to logs/<function name>.EPOCH_TIME"
-# '2>&1 /dev/null' means "hide STDOUT and STDERR, no logging"
-# '>> ${BASEDIR}/logs/${FUNCNAME}.$(date +%s) 2>&1' means "log both STDOUT and STDERR to logs/<function name>.EPOCH_TIME, no output"
-# '>> ${BASEDIR}/logs/${FUNCNAME}.$(date +%s)' means "log STDOUT to logs/<function name>.EPOCH_TIME, display (but don't log) STDERR)"
-# '' means "no logging; display both STDOUT and STDERR"

extra/pre-build.d/root/pre-build.sh

@@ -145,3 +145,8 @@ then
 	apacman --gendb
 	cleanPacorigs
 fi
+# Cleanup
+yes | pacman -Scc
+rm -f /root/.bash_history
+rm -f /root/.viminfo
+

extra/templates/BIOS/isolinux.cfg.arch.j2

@@ -0,0 +1,73 @@
+UI vesamenu.c32
+DEFAULT check
+PROMPT 0
+TIMEOUT 50
+MENU HIDDEN
+ONTIMEOUT {{ bdisk['uxname'] }}_ram_{{ bitness }}
+MENU TABMSG Press [TAB] to edit options
+#MENU TITLE {{ bdisk['pname'] }} (ISO edition)
+MENU ROWS 16
+MENU TIMEOUTROW 22
+MENU TABMSGROW 24
+MENU CMDLINEROW 24
+MENU HELPMSGROW 26
+MENU WIDTH 78
+MENU MARGIN 6
+MENU IMMEDIATE
+# http://www.colorpicker.com/
+MENU color border   0 	        #00000000 #00000000 none
+MENU color title    0    	#FFF5B800 #00000000 std 
+MENU color sel      7;37;40    #FF000000 #FFFFFFFF all 
+MENU color hotsel   1;7;37;40  #FFFF0000 #FFC0C0C0 all 
+MENU color hotkey   1;7;37;40  #FF0000CC #FFC0C0C0 all 
+MENU color tabmsg   1;31;40    #FF808080 #00000000 std 
+MENU color help     1;31;40    #FFFFFFFF #FF000000 none
+MENU color timeout_msg 0       #FFFFB300 #00000000 none
+MENU color timeout  0          #FFFF0000 #FF000000 none
+MENU color cmdline  0          #FFFFFFFF #FF000000 none
+MENU color cmdmark  1;36;40    #C000FFFF #FF000000 std 
+MENU color scrollbar 30;44     #FF00FF00 #FF000000 std 
+MENU color msg07    0          #FF000000 #00FFFFFF none
+MENU BACKGROUND /{{ bdisk['uxname'] }}.png
+
+LABEL local_override
+  MENU LABEL Local ^Boot
+  localboot 0
+  TEXT HELP
+  Boot from the local system instead.
+  ENDTEXT 
+
+LABEL reboot
+  MENU LABEL ^Reboot
+  COM32 reboot.c32
+  TEXT HELP
+  Reboot the machine
+  ENDTEXT
+
+MENU SEPARATOR
+
+MENU BEGIN {{ bitness }}BIT
+MENU LABEL ^1) {{ bitness }}-Bit ...
+ONTIMEOUT {{ bdisk['uxname'] }}_{{ bitness }}
+  
+LABEL {{ bdisk['uxname'] }}_ram_{{ bitness }}
+  MENU LABEL ^1) {{ bdisk['pname'] }} (run from RAM)
+  LINUX /boot/{{ bdisk['uxname'] }}.{{ bitness }}.kern
+  INITRD /boot/{{ bdisk['uxname'] }}.{{ bitness }}.img
+  APPEND copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
+  TEXT HELP
+  {{ bitness }}-bit, run from RAM
+  ENDTEXT
+  MENU DEFAULT
+
+LABEL {{ bdisk['uxname'] }}_{{ bitness }} 
+  MENU LABEL ^1) {{ bdisk['pname'] }} (Default)
+  LINUX /boot/{{ bdisk['uxname'] }}.{{ bitness }}.kern
+  INITRD /boot/{{ bdisk['uxname'] }}.{{ bitness }}.img
+  APPEND archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
+  TEXT HELP
+  Same as the above, except run directly from the CD-
+  don't copy the image to RAM. (Best for lower-memory boxes)
+  ENDTEXT
+
+MENU END

extra/templates/BIOS/isolinux.cfg.multi.j2

@@ -0,0 +1,110 @@
+UI vesamenu.c32
+DEFAULT check
+PROMPT 0
+TIMEOUT 50
+MENU HIDDEN
+#ONTIMEOUT {{ bdisk['uxname'] }}_ram
+ONTIMEOUT check
+MENU TABMSG Press [TAB] to edit options
+#MENU TITLE {{ bdisk['pname'] }} (ISO edition)
+MENU ROWS 16
+MENU TIMEOUTROW 22
+MENU TABMSGROW 24
+MENU CMDLINEROW 24
+MENU HELPMSGROW 26
+MENU WIDTH 78
+MENU MARGIN 6
+MENU IMMEDIATE
+# http://www.colorpicker.com/
+MENU color border   0 	        #00000000 #00000000 none
+MENU color title    0    	#FFF5B800 #00000000 std 
+MENU color sel      7;37;40    #FF000000 #FFFFFFFF all 
+MENU color hotsel   1;7;37;40  #FFFF0000 #FFC0C0C0 all 
+MENU color hotkey   1;7;37;40  #FF0000CC #FFC0C0C0 all 
+MENU color tabmsg   1;31;40    #FF808080 #00000000 std 
+MENU color help     1;31;40    #FFFFFFFF #FF000000 none
+MENU color timeout_msg 0       #FFFFB300 #00000000 none
+MENU color timeout  0          #FFFF0000 #FF000000 none
+MENU color cmdline  0          #FFFFFFFF #FF000000 none
+MENU color cmdmark  1;36;40    #C000FFFF #FF000000 std 
+MENU color scrollbar 30;44     #FF00FF00 #FF000000 std 
+MENU color msg07    0          #FF000000 #00FFFFFF none
+MENU BACKGROUND /{{ bdisk['uxname'] }}.png
+
+LABEL check
+  MENU LABEL Your best supported kernel should be detected automatically.
+  COM32 ifcpu64.c32
+  APPEND {{ bdisk['uxname'] }}_64 -- {{ bdisk['uxname'] }}_32
+  MENU DEFAULT
+
+
+
+LABEL local_override
+  MENU LABEL Local ^Boot
+  localboot 0
+  TEXT HELP
+  Boot from the local system instead.
+  ENDTEXT 
+
+LABEL reboot
+  MENU LABEL ^Reboot
+  COM32 reboot.c32
+  TEXT HELP
+  Reboot the machine
+  ENDTEXT
+
+MENU SEPARATOR
+
+## 64 BIT
+MENU BEGIN 64BIT
+MENU LABEL ^1) 64-Bit ...
+ONTIMEOUT {{ bdisk['uxname'] }}_64
+  
+LABEL {{ bdisk['uxname'] }}_ram_64
+  MENU LABEL ^1) {{ bdisk['pname'] }} (run from RAM)
+  LINUX /boot/{{ bdisk['uxname'] }}.64.kern
+  INITRD /boot/{{ bdisk['uxname'] }}.64.img
+  APPEND copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
+  TEXT HELP
+  64-bit, run from RAM
+  ENDTEXT
+  MENU DEFAULT
+
+LABEL {{ bdisk['uxname'] }}_64 
+  MENU LABEL ^1) {{ bdisk['pname'] }} (Default)
+  LINUX /boot/{{ bdisk['uxname'] }}.64.kern
+  INITRD /boot/{{ bdisk['uxname'] }}.64.img
+  APPEND archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
+  TEXT HELP
+  Same as the above, except run directly from the CD-
+  don't copy the image to RAM. (Best for lower-memory boxes)
+  ENDTEXT
+
+MENU END
+
+MENU BEGIN 32BIT
+MENU LABEL ^2) 32-Bit ...
+ONTIMEOUT {{ bdisk['uxname'] }}_32
+
+## 32 BIT
+LABEL {{ bdisk['uxname'] }}_ram_32
+  MENU LABEL ^1) {{ bdisk['pname'] }} (run from RAM)
+  LINUX /boot/{{ bdisk['uxname'] }}.32.kern
+  INITRD /boot/{{ bdisk['uxname'] }}.32.img
+  APPEND copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
+  TEXT HELP
+  32-bit, run from RAM
+  ENDTEXT
+  MENU DEFAULT
+
+LABEL {{ bdisk['uxname'] }}_32 
+  MENU LABEL ^2) {{ bdisk['pname'] }} (Default)
+  LINUX /boot/{{ bdisk['uxname'] }}.32.kern
+  INITRD /boot/{{ bdisk['uxname'] }}.32.img
+  APPEND archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}
+  TEXT HELP
+  Same as the above, except run directly from the CD-
+  don't copy the image to RAM. (Best for lower-memory boxes)
+  ENDTEXT
+
+MENU END

extra/templates/EFI/base.conf.j2

@@ -0,0 +1,4 @@
+title   {{ bdisk['pname'] }} (Media)
+linux   /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.{% if efi is defined %}efi{% else %}kern{% endif %}
+initrd  /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.img
+options archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}

extra/templates/EFI/loader.conf.j2

@@ -0,0 +1,2 @@
+timeout 3
+default {{ bdisk['uxname'] }}

extra/templates/EFI/ram.conf.j2

@@ -0,0 +1,4 @@
+title   {{ bdisk['pname'] }} (RAM)
+linux   /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.{% if efi is defined %}efi{% else %}kern{% endif %}
+initrd  /{% if efi is defined %}EFI/{{ bdisk['name'] }}{% else $}boot{% endif %}/{{ bdisk['uxname'] }}.img
+options copytoram archisobasedir={{ bdisk['name'] }} archisolabel={{ bdisk['name'] }}

extra/templates/EFI/uefi1.conf.j2

@@ -0,0 +1,2 @@
+title   UEFI Shell (v1)
+efi    /EFI/shellx64_v1.efi

extra/templates/EFI/uefi2.conf.j2

@@ -0,0 +1,2 @@
+title   UEFI Shell (v2)
+efi    /EFI/shellx64_v2.efi

extra/templates/VERSION_INFO.txt.j2

@@ -3,3 +3,4 @@ Build:          {{ build['name'] }}
 Time:           {{ build['time'] }}
 Machine:        {{ hostname }}
 User:           {{ build['user'] }}{% if build['realuser'] is defined and build['realuser'] > 0 %} ({{ build['realuser'] }}){% endif %}
+

extra/templates/iPXE/EMBED.j2

@@ -0,0 +1,8 @@
+#!ipxe
+
+dhcp
+## TODO: signed kernel and initrd
+#imgtrust --permanent
+#imgverify vmlinuz path/to/vmlinuz.sig
+#imgverify initrd path/to/initrd.sig
+chain {{ ipxe['uri'] }}

overlay/etc/systemd/system/getty.target.wants/getty@tty1.service

@@ -1 +0,0 @@
-/usr/lib/systemd/system/getty@.service