checking in- currently trying to get the cleanup sorted. i need something that works better, i think
This commit is contained in:
205
bdisk/prep.py
205
bdisk/prep.py
@@ -10,107 +10,115 @@ import git
|
||||
import jinja2
|
||||
import datetime
|
||||
from urllib.request import urlopen
|
||||
|
||||
def archChk(arch):
|
||||
if arch in ['i686', 'x86_64']:
|
||||
return(arch)
|
||||
else:
|
||||
exit("{0} is not a valid architecture. Must be one of i686 or x86_64.".format(arch))
|
||||
import host # bdisk.host
|
||||
|
||||
def dirChk(config_dict):
|
||||
# Make dirs if they don't exist
|
||||
for d in ('archboot', 'isodir', 'mountpt', 'srcdir', 'tempdir'):
|
||||
os.makedirs(config_dict['build'][d], exists_ok = True)
|
||||
os.makedirs(config_dict['build'][d], exist_ok = True)
|
||||
# Make dirs for sync staging if we need to
|
||||
for x in ('http', 'tftp'):
|
||||
if config_dict['sync'][x]:
|
||||
os.makedirs(config_dict[x]['path'], exist_ok = True)
|
||||
|
||||
def downloadTarball(arch, dlpath):
|
||||
# arch - should be i686 or x86_64
|
||||
def downloadTarball(build):
|
||||
dlpath = build['dlpath']
|
||||
arch = build['arch']
|
||||
#mirror = 'http://mirrors.kernel.org/archlinux'
|
||||
mirror = build['mirrorproto'] + '://' + build['mirror']
|
||||
rlsdir = mirror + build['mirrorpath']
|
||||
sha_in = urlopen(mirror + build['mirrorchksum'])
|
||||
# returns path/filename e.g. /some/path/to/file.tar.gz
|
||||
# we use .gnupg since we'll need it later.
|
||||
archChk(arch)
|
||||
try:
|
||||
os.makedirs(dlpath + '/.gnupg')
|
||||
except OSError as exception:
|
||||
if exception.errno != errno.EEXIST:
|
||||
raise
|
||||
tarball_path = dlpath + '/.latest.' + arch + '.tar.gz'
|
||||
#mirror = 'http://mirrors.kernel.org/archlinux'
|
||||
mirror = 'https://mirror.us.leaseweb.net/archlinux'
|
||||
rlsdir = mirror + '/iso/latest'
|
||||
sha_in = urlopen(rlsdir + '/sha1sums.txt')
|
||||
os.makedirs(dlpath + '/.gnupg', exist_ok = True)
|
||||
tarball_path = {}
|
||||
for x in arch:
|
||||
tarball_path[x] = dlpath + '/.latest.' + x + '.tar'
|
||||
sha1sums = sha_in.read()
|
||||
sha_in.close()
|
||||
sha1_list = sha1sums.decode("utf-8")
|
||||
sha_list = list(filter(None, sha1_list.split('\n')))
|
||||
sha_raw = sha1sums.decode("utf-8")
|
||||
sha_list = list(filter(None, sha_raw.split('\n')))
|
||||
sha_dict = {x.split()[1]: x.split()[0] for x in sha_list}
|
||||
pattern = re.compile('^archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-' + arch + '\.tar\.gz$')
|
||||
tarball = [filename.group(0) for l in list(sha_dict.keys()) for filename in [pattern.search(l)] if filename][0]
|
||||
sha1 = sha_dict[tarball]
|
||||
# all that lousy work just to get a sha1 sum. okay. so.
|
||||
if os.path.isfile(tarball_path):
|
||||
pass
|
||||
else:
|
||||
# fetch the tarball...
|
||||
print("Fetching the tarball for {0} architecture, please wait...".format(arch))
|
||||
tarball_dl = urlopen(rlsdir + tarball)
|
||||
with open(dlpath + '/latest.' + arch + '.tar.gz', 'wb') as f:
|
||||
f.write(tarball_dl)
|
||||
tarball_dl.close()
|
||||
tarball_hash = hashlib.sha1(open(tarball_path, 'rb').read()).hexdigest()
|
||||
if tarball_hash != sha1:
|
||||
exit("There was a failure fetching the tarball and the wrong version exists on the filesystem.\nPlease try again later.")
|
||||
else:
|
||||
# okay, so the sha1 matches. let's verify the signature.
|
||||
if build['mirrorgpgsig'] != '':
|
||||
# we don't want to futz with the user's normal gpg.
|
||||
gpg = gnupg.GPG(gnupghome = dlpath + '/.gnupg')
|
||||
print("\nNow generating a GPG key. Please wait...")
|
||||
# python-gnupg 0.3.9 spits this error in Arch. it's harmless, but ugly af.
|
||||
# TODO: remove this when the error doesn't happen anymore.
|
||||
print("If you see a \"ValueError: Unknown status message: 'KEY_CONSIDERED'\" error, it can be safely ignored.")
|
||||
print("If this is taking a VERY LONG time, try installing haveged and starting it. This can be " +
|
||||
"done safely in parallel with the build process.\n")
|
||||
input_data = gpg.gen_key_input(name_email = 'tempuser@nodomain.tld', passphrase = 'placeholder_passphrase')
|
||||
key = gpg.gen_key(input_data)
|
||||
keyid = '7F2D434B9741E8AC'
|
||||
gpg.recv_keys('pgp.mit.edu', keyid)
|
||||
gpg_sig = tarball + '.sig'
|
||||
sig_dl = urlopen(rlsdir + gpg_sig)
|
||||
with open(tarball_path + '.sig', 'wb+') as f:
|
||||
f.write(sig_dl)
|
||||
sig_dl.close()
|
||||
sig = tarball_path + '.sig'
|
||||
tarball_data = open(tarball_path, 'rb')
|
||||
tarball_data_in = tarball_data.read()
|
||||
gpg_verify = gpg.verify_data(sig, tarball_data_in)
|
||||
tarball_data.close()
|
||||
if not gpg_verify:
|
||||
exit("There was a failure checking the signature of the release tarball. Please investigate.")
|
||||
os.remove(sig)
|
||||
keyid = build['gpgkey']
|
||||
gpg.recv_keys(build['gpgkeyserver'], keyid)
|
||||
for a in arch:
|
||||
pattern = re.compile('^.*' + a + '\.tar(\.(gz|bz2|xz))?$')
|
||||
tarball = [filename.group(0) for l in list(sha_dict.keys()) for filename in [pattern.search(l)] if filename][0]
|
||||
sha1 = sha_dict[tarball]
|
||||
if os.path.isfile(tarball_path[a]):
|
||||
pass
|
||||
else:
|
||||
# fetch the tarball...
|
||||
print("Fetching the tarball for {0} architecture, please wait...".format(a))
|
||||
#dl_file = urllib.URLopener()
|
||||
tarball_dl = urlopen(rlsdir + tarball)
|
||||
with open(tarball_path[a], 'wb') as f:
|
||||
f.write(tarball_dl.read())
|
||||
tarball_dl.close()
|
||||
print(("Checking that the hash checksum for {0} matches {1}, please wait...").format(
|
||||
tarball_path[a], sha1))
|
||||
tarball_hash = hashlib.sha1(open(tarball_path[a], 'rb').read()).hexdigest()
|
||||
if tarball_hash != sha1:
|
||||
exit(("There was a failure fetching {0} and the wrong version exists on the filesystem.\n" +
|
||||
"Please try again later.").format(tarball))
|
||||
elif build['mirrorgpgsig'] != '':
|
||||
# okay, so the sha1 matches. let's verify the signature.
|
||||
if build['mirrorgpgsig'] == '.sig':
|
||||
gpgsig_remote = rlsdir + tarball + '.sig'
|
||||
else:
|
||||
gpgsig_remote = mirror + build['mirrorgpgsig']
|
||||
gpg_sig = tarball + '.sig'
|
||||
sig_dl = urlopen(gpgsig_remote)
|
||||
sig = tarball_path[a] + '.sig'
|
||||
with open(sig, 'wb+') as f:
|
||||
f.write(sig_dl.read())
|
||||
sig_dl.close()
|
||||
tarball_data = open(tarball_path[a], 'rb')
|
||||
tarball_data_in = tarball_data.read()
|
||||
gpg_verify = gpg.verify_data(sig, tarball_data_in)
|
||||
tarball_data.close()
|
||||
if not gpg_verify:
|
||||
exit("There was a failure checking {0} against {1}. Please investigate.".format(
|
||||
sig, tarball_path[a]))
|
||||
os.remove(sig)
|
||||
|
||||
return(tarball_path)
|
||||
|
||||
def unpackTarball(tarball_path, chrootdir):
|
||||
def unpackTarball(tarball_path, build):
|
||||
chrootdir = build['chrootdir']
|
||||
# Make the dir if it doesn't exist
|
||||
try:
|
||||
os.makedirs(chrootdir)
|
||||
except OSError as exception:
|
||||
if exception.errno != errno.EEXIST:
|
||||
raise
|
||||
shutil.rmtree(chrootdir, ignore_errors = True)
|
||||
os.makedirs(chrootdir, exist_ok = True)
|
||||
print("Now extracting the tarball(s). Please wait...")
|
||||
# Open and extract the tarball
|
||||
tar = tarfile.open(tarball_path, 'r:gz')
|
||||
tar.extractall(path = destdir)
|
||||
tar.close()
|
||||
return(True)
|
||||
for a in build['arch']:
|
||||
tar = tarfile.open(tarball_path[a], 'r:gz')
|
||||
tar.extractall(path = chrootdir)
|
||||
tar.close()
|
||||
print("Extraction for {0} finished.".format(tarball_path[a]))
|
||||
|
||||
def buildChroot(arch, chrootdir, dlpath, extradir):
|
||||
unpack_me = unpackTarball(downloadTarball(archChk(arch), dlpath), chrootdir)
|
||||
if unpack_me:
|
||||
pass
|
||||
else:
|
||||
exit("Something went wrong when trying to unpack the tarball.")
|
||||
|
||||
print("The download and extraction has completed. Now prepping the chroot environment with some additional changes.")
|
||||
def buildChroot(build):
|
||||
dlpath = build['dlpath']
|
||||
chrootdir = build['chrootdir']
|
||||
arch = build['arch']
|
||||
extradir = build['basedir'] + '/extra'
|
||||
unpack_me = unpackTarball(downloadTarball(build), build)
|
||||
# build dict of lists of files and dirs from pre-build.d dir, do the same with arch-specific changes.
|
||||
prebuild_overlay = {}
|
||||
prebuild_arch_overlay = {}
|
||||
for x in ['i686', 'x86_64']:
|
||||
for x in arch:
|
||||
prebuild_arch_overlay[x] = {}
|
||||
for y in ['files', 'dirs']:
|
||||
prebuild_overlay[y] = []
|
||||
@@ -129,40 +137,45 @@ def buildChroot(arch, chrootdir, dlpath, extradir):
|
||||
prebuild_overlay[x][:] = [y for y in prebuild_overlay[x] if not y.startswith(('x86_64','i686'))]
|
||||
prebuild_overlay['dirs'].remove('/')
|
||||
# create the dir structure. these should almost definitely be owned by root.
|
||||
for dir in prebuild_overlay['dirs']:
|
||||
os.makedirs(chrootdir + '/' + dir, exist_ok = True)
|
||||
os.chown(chrootdir + '/' + dir, 0, 0)
|
||||
# and copy over the files. again, chown to root.
|
||||
for file in prebuild_overlay['files']:
|
||||
shutil.copy2(extradir + '/pre-build.d/' + file, chrootdir + '/' + file)
|
||||
os.chown(chrootdir + '/' + file, 0, 0)
|
||||
# do the same for arch-specific stuff.
|
||||
for dir in prebuild_arch_overlay[arch]['dirs']:
|
||||
os.makedirs(chrootdir + '/' + dir, exist_ok = True)
|
||||
os.chown(chrootdir + '/' + dir, 0, 0)
|
||||
for file in prebuild_arch_overlay[arch]['files']:
|
||||
shutil.copy2(extradir + '/pre-build.d/' + arch + '/' + file, chrootdir + '/' + file)
|
||||
os.chown(chrootdir + '/' + file, 0, 0)
|
||||
return(chrootdir)
|
||||
for a in arch:
|
||||
for dir in prebuild_overlay['dirs']:
|
||||
os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
|
||||
os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0)
|
||||
# and copy over the files. again, chown to root.
|
||||
for file in prebuild_overlay['files']:
|
||||
shutil.copy2(extradir + '/pre-build.d/' + file, chrootdir + '/root.' + a + '/' + file)
|
||||
os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0)
|
||||
# do the same for arch-specific stuff.
|
||||
for dir in prebuild_arch_overlay[a]['dirs']:
|
||||
os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True)
|
||||
os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0)
|
||||
for file in prebuild_arch_overlay[a]['files']:
|
||||
shutil.copy2(extradir + '/pre-build.d/' + a + '/' + file, chrootdir + '/root.' + a + '/' + file)
|
||||
os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0)
|
||||
|
||||
def prepChroot(templates_dir, chrootdir, bdisk, arch):
|
||||
def prepChroot(templates_dir, build, bdisk):
|
||||
chrootdir = build['chrootdir']
|
||||
arch = build['arch']
|
||||
bdisk_repo_dir = build['basedir']
|
||||
build = {}
|
||||
# let's prep some variables to write out the version info.txt
|
||||
# get the git tag and short commit hash
|
||||
repo = git.Repo(bdisk['dir'])
|
||||
repo = git.Repo(bdisk_repo_dir)
|
||||
refs = repo.git.describe(repo.head.commit).split('-')
|
||||
build['ver'] = refs[0] + '-' + refs[2]
|
||||
# and these should be passed in from the args, from the most part.
|
||||
build['name'] = bdisk['name']
|
||||
build['time'] = datetime.datetime.utcnow().strftime("%a %b %d %H:%M:%S UTC %Y")
|
||||
build['host'] = bdisk['hostname']
|
||||
hostname = host.getHostname
|
||||
build['user'] = os.environ['USER']
|
||||
if os.environ['SUDO_USER']:
|
||||
if 'SUDO_USER' in os.environ:
|
||||
build['realuser'] = os.environ['SUDO_USER']
|
||||
# and now that we have that dict, let's write out the VERSION_INFO.txt file.
|
||||
env = jinja2.Environment(loader=FileSystemLoader(templates_dir))
|
||||
loader = jinja2.FileSystemLoader(templates_dir)
|
||||
env = jinja2.Environment(loader = loader)
|
||||
tpl = env.get_template('VERSION_INFO.txt.j2')
|
||||
tpl_out = template.render(build = build)
|
||||
with open(chrootdir + '/root/VERSION_INFO.txt', "wb+") as f:
|
||||
fh.write(tpl_out)
|
||||
tpl_out = tpl.render(build = build, hostname = hostname)
|
||||
for a in arch:
|
||||
with open(chrootdir + '/root.' + a + '/root/VERSION_INFO.txt', "w+") as f:
|
||||
f.write(tpl_out)
|
||||
return(build)
|
||||
|
||||
Reference in New Issue
Block a user