checking in...

This commit is contained in:
brent s. 2018-05-22 06:01:18 -04:00
parent 1d9b40a597
commit 4de9d1a26c
8 changed files with 426 additions and 148 deletions

25
TODO
View File

@ -1,24 +1,23 @@
- write classes/functions - write classes/functions
- XML-based config - XML-based config
-x XML syntax
--- regex btags - case-insensitive? this can be represented in-pattern:
https://stackoverflow.com/a/9655186/733214
-x configuration generator
--- print end result xml config to stderr for easier redirection? or print prompts to stderr and xml to stdout?
-- XSD for validation
-- Flask app for generating config?
-- TKinter (or pygame?) GUI?
--- https://docs.python.org/3/faq/gui.html
--- https://www.pygame.org/wiki/gui
- ensure we use docstrings in a Sphinx-compatible manner? - ensure we use docstrings in a Sphinx-compatible manner?
https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
at the very least document all the functions and such so pydoc's happy. at the very least document all the functions and such so pydoc's happy.
- better prompt display. i might include them as constants in a separate file
and then import it for e.g. confgen. or maybe a Flask website/app?
- locking - locking
- for docs, 3.x (as of 3.10) was 2.4M. - for docs, 3.x (as of 3.10) was 2.4M.
- GUI? at least for generating config...
- Need ability to write/parse mtree specs (or a similar equivalent) for applying ownerships/permissions to overlay files - Need ability to write/parse mtree specs (or a similar equivalent) for applying ownerships/permissions to overlay files


- SSL key gen:
import OpenSSL
k = OpenSSL.crypto.PKey()
k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096)
x = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
k,
cipher = 'aes256',
passphrase = 'test')

- need to package: - need to package:
python-hashid (https://psypanda.github.io/hashID/, python-hashid (https://psypanda.github.io/hashID/,
https://github.com/psypanda/hashID, https://github.com/psypanda/hashID,
@ -41,4 +40,4 @@ BUGS.SQUARE-R00T.NET bugs/tasks:
#36: Allow parsing pkg lists with inline comments #36: Allow parsing pkg lists with inline comments
#39: Fix UEFI #39: Fix UEFI
#40: ISO overlay (to add e.g. memtest86+ to final ISO) #40: ISO overlay (to add e.g. memtest86+ to final ISO)
#43: Support resuming partial tarball downloads (Accet-Ranges: bytes) #43: Support resuming partial tarball downloads (Accept-Ranges: bytes)

View File

@ -3,3 +3,10 @@ import OpenSSL
# migrate old functions of bSSL to use cryptography # migrate old functions of bSSL to use cryptography
# but still waiting on their recpipes. # but still waiting on their recpipes.
# https://cryptography.io/en/latest/x509/tutorial/ # https://cryptography.io/en/latest/x509/tutorial/
#import OpenSSL
#k = OpenSSL.crypto.PKey()
#k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096)
#x = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
# k,
# cipher = 'aes256',
# passphrase = 'test')

View File

@ -442,6 +442,8 @@ class ConfGenerator(object):
tarball_elem.attrib['flags'] = 'latest' tarball_elem.attrib['flags'] = 'latest'
tarball_elem.text = tarball['full_url'] tarball_elem.text = tarball['full_url']
print('\n++ SOURCES || {0} || CHECKSUM ++'.format(arch.upper())) print('\n++ SOURCES || {0} || CHECKSUM ++'.format(arch.upper()))
# TODO: explicit not being set for explicitly-set sums,
# and checksum string not actually added to those. investigate.
chksum = lxml.etree.SubElement(source, 'checksum') chksum = lxml.etree.SubElement(source, 'checksum')
_chksum_chk = prompt.confirm_or_no(prompt = ( _chksum_chk = prompt.confirm_or_no(prompt = (
'\nWould you like to add a checksum for the tarball? (BDisk ' '\nWould you like to add a checksum for the tarball? (BDisk '
@ -502,7 +504,7 @@ class ConfGenerator(object):
print('Invalid selection. Starting over.') print('Invalid selection. Starting over.')
continue continue
else: else:
checksum_type == checksum_type[0] checksum_type = checksum_type[0]
chksum.attrib['explicit'] = "yes" chksum.attrib['explicit'] = "yes"
chksum.text = checksum chksum.text = checksum
chksum.attrib['hash_algo'] = checksum_type chksum.attrib['hash_algo'] = checksum_type

View File

@ -1,15 +1,18 @@
import copy
import re
import os import os
import pprint
import re
import utils import utils
import validators
import lxml.etree import lxml.etree
from urllib.parse import urlparse from urllib.parse import urlparse


etree = lxml.etree etree = lxml.etree
detect = utils.detect()
generate = utils.generate()
transform = utils.transform()
valid = utils.valid()


class Conf(object): class Conf(object):
def __init__(self, cfg, profile = None): def __init__(self, cfg, profile = None, validate = False):
""" """
A configuration object. A configuration object.


@ -37,27 +40,70 @@ class Conf(object):
You can provide any combination of these You can provide any combination of these
(e.g. "profile={'id': 2, 'name' = 'some_profile'}"). (e.g. "profile={'id': 2, 'name' = 'some_profile'}").
""" """
#self.raw = _detect_cfg(cfg) # no longer needed; in utils
self.xml_suppl = utils.xml_supplicant(cfg, profile = profile) self.xml_suppl = utils.xml_supplicant(cfg, profile = profile)
self.profile = self.xml_suppl self.xml = self.xml_suppl.xml
self.xml = None for e in self.xml_suppl.xml.iter():
self.profile = None self.xml_suppl.substitute(e)
# Mad props to https://stackoverflow.com/a/12728199/733214 self.xml_suppl.get_profile(profile = self.xml_suppl.orig_profile)
self.xpath_re = re.compile('(?<=(?<!\{)\{)[^{}]*(?=\}(?!\}))') with open('/tmp/parsed.xml', 'wb') as f:
self.substitutions = {} f.write(lxml.etree.tostring(self.xml_suppl.xml))
self.xpaths = ['xpath'] self.profile = self.xml_suppl.profile
try:
self.xml = etree.fromstring(self.raw)
except lxml.etree.XMLSyntaxError:
raise ValueError('The configuration provided does not seem to be '
'valid')
self.xsd = None self.xsd = None
self.cfg = {}
#if validate:
#if not self.validate(): # Need to write the XSD #if not self.validate(): # Need to write the XSD
# raise ValueError('The configuration did not pass XSD/schema ' # raise ValueError('The configuration did not pass XSD/schema '
# 'validation') # 'validation')
self.get_profile()
self.max_recurse = int(self.profile.xpath('//meta/' def get_source(self, source, item, _source):
'max_recurse')[0].text) _source_item = {'flags': [],
'fname': None}
elem = source.xpath('./{0}'.format(item))[0]
if item == 'checksum':
if elem.get('explicit', False):
_explicit = transform.xml2py(
elem.attrib['explicit'])
_source_item['explicit'] = _explicit
if _explicit:
del(_source_item['fname'])
_source_item['value'] = elem.text
return(_source_item)
else:
_source_item['explicit'] = False
if elem.get('hash_algo', False):
_source_item['hash_algo'] = elem.attrib['hash_algo']
else:
_source_item['hash_algo'] = None
if item == 'sig':
if elem.get('keys', False):
_keys = [i.strip() for i in elem.attrib['keys'].split(',')]
_source_item['keys'] = _keys
else:
_source_item['keys'] = []
if elem.get('keyserver', False):
_source_item['keyserver'] = elem.attrib['keyserver']
else:
_source_item['keyserver'] = None
_item = elem.text
_flags = elem.get('flags', [])
if _flags:
for f in _flags.split(','):
if f.strip().lower() == 'none':
continue
_source_item['flags'].append(f.strip().lower())
if _source_item['flags']:
if 'regex' in _source_item['flags']:
ptrn = _item.format(**self.xml_suppl.btags['regex'])
else:
ptrn = None
_source_item['fname'] = detect.remote_files(
'/'.join((_source['mirror'],
_source['rootpath'])),
ptrn = ptrn,
flags = _source_item['flags'])
else:
_source_item['fname'] = _item
return(_source_item)


def get_xsd(self): def get_xsd(self):
path = os.path.join(os.path.dirname(__file__), path = os.path.join(os.path.dirname(__file__),
@ -66,62 +112,116 @@ class Conf(object):
xsd = f.read() xsd = f.read()
return(xsd) return(xsd)


def parse_accounts(self):
## PROFILE/ACCOUNTS
self.cfg['users'] = []
# First we handle the root user, since it's a "special" case.
_root = self.profile.xpath('./accounts/rootpass')
self.cfg['root'] = transform.user(_root)
for user in self.profile.xpath('./accounts/user'):
_user = {'username': user.xpath('./username/text()')[0],
'sudo': transform.xml2py(user.attrib['sudo']),
'comment': None}
_comment = user.xpath('./comment/text()')
if len(_comment):
_user['comment'] = _comment[0]
_password = user.xpath('./password')
_user.update(transform.user(_password))
self.cfg['users'].append(_user)
return()

def parse_all(self):
self.parse_profile()
self.parse_meta()
self.parse_accounts()
self.parse_sources()
self.parse_buildpaths()
self.parse_pki()
return()

def parse_buildpaths(self):
## PROFILE/BUILD(/PATHS)
self.cfg['build'] = {'paths': {}}
build = self.profile.xpath('./build')[0]
_optimize = build.get('its_full_of_stars', 'no')
self.cfg['build']['optimize'] = transform.xml2py(_optimize)
for path in build.xpath('./paths/*'):
self.cfg['build']['paths'][path.tag] = path.text
self.cfg['build']['basedistro'] = build.get('basedistro', 'archlinux')
# iso and ipxe are their own basic profile elements, but we group them
# in here because 1.) they're related, and 2.) they're simple to
# import. This may change in the future if they become more complex.
## PROFILE/ISO
self.cfg['iso'] = {'sign': None,
'multi_arch': None}
self.cfg['ipxe'] = {'sign': None,
'iso': None}
for x in ('iso', 'ipxe'):
# We enable all features by default.
elem = self.profile.xpath('./{0}'.format(x))[0]
for a in self.cfg[x]:
self.cfg[x][a] = transform.xml2py(elem.get(a, 'yes'))
if x == 'ipxe':
self.cfg[x]['uri'] = elem.xpath('./uri/text()')[0]
return()

def parse_meta(self):
## PROFILE/META
# Get the various meta strings. We skip regexes (we handle those
# separately since they're unique'd per id attrib) and variables (they
# are already substituted by self.xml_suppl.substitute(x)).
_meta_iters = ('dev', 'names')
for t in _meta_iters:
self.cfg[t] = {}
_xpath = './meta/{0}'.format(t)
for e in self.profile.xpath(_xpath):
for se in e:
if not isinstance(se, lxml.etree._Comment):
self.cfg[t][se.tag] = se.text
for e in ('desc', 'uri', 'ver', 'max_recurse'):
_xpath = './meta/{0}/text()'.format(e)
self.cfg[e] = self.profile.xpath(_xpath)[0]
# HERE is where we would handle regex patterns.
# But we don't, because they're in self.xml_suppl.btags['regex'].
#self.cfg['regexes'] = {}
#_regexes = self.profile.xpath('./meta/regexes/pattern')
#if len(_regexes):
# for ptrn in _regexes:
# self.cfg['regexes'][ptrn.attrib['id']] = re.compile(ptrn.text)
return()

def parse_pki(self):
self.cfg['pki'] = {'ca': {},
'client': []}
elem = self.profile.xpath('./pki')[0]
self.cfg['pki']['overwrite'] =

def parse_profile(self):
## PROFILE
# The following are attributes of profiles that serve as identifiers.
self.cfg['profile'] = {'id': None,
'name': None,
'uuid': None}
for a in self.cfg['profile']:
if a in self.profile.attrib:
self.cfg['profile'][a] = self.profile.attrib[a]
return()

def parse_sources(self):
## PROFILE/SOURCES
self.cfg['sources'] = []
for source in self.profile.xpath('./sources/source'):
_source = {}
_source['arch'] = source.attrib['arch']
_source['mirror'] = source.xpath('./mirror/text()')[0]
_source['rootpath'] = source.xpath('./rootpath/text()')[0]
# The tarball, checksum, and sig components requires some...
# special care.
for e in ('tarball', 'checksum', 'sig'):
_source[e] = self.get_source(source, e, _source)
self.cfg['sources'].append(_source)
return()

def validate(self): def validate(self):
self.xsd = etree.XMLSchema(self.get_xsd()) self.xsd = etree.XMLSchema(self.get_xsd())
return(self.xsd.validate(self.xml)) return(self.xsd.validate(self.xml))

def get_profile(self):
"""Get a configuration profile.

Get a configuration profile from the XML object and set that as a
profile object. If a profile is specified, attempt to find it. If not,
follow the default rules as specified in __init__.
"""
if self.profile:
# A profile identifier was provided
if isinstance(self.profile, str):
_profile_name = self.profile
self.profile = {}
for i in _profile_specifiers:
self.profile[i] = None
self.profile['name'] = _profile_name
elif isinstance(self.profile, dict):
for k in _profile_specifiers:
if k not in self.profile.keys():
self.profile[k] = None
else:
raise TypeError('profile must be a string (name of profile), '
'a dictionary, or None')
xpath = ('/bdisk/'
'profile{0}').format(_profile_xpath_gen(self.profile))
self.profile = self.xml.xpath(xpath)
if not self.profile:
raise RuntimeError('Could not find the profile specified in '
'the given configuration')
else:
# We need to find the default.
profiles = []
for p in self.xml.xpath('/bdisk/profile'):
profiles.append(p)
# Look for one named "default" or "DEFAULT" etc.
for idx, value in enumerate([e.attrib['name'].lower() \
for e in profiles]):
if value == 'default':
self.profile = copy.deepcopy(profiles[idx])
break
# We couldn't find a profile with a default name. Try to grab the
# first profile.
if self.profile is None:
# Grab the first profile.
if profiles:
self.profile = profile[0]
else:
# No profiles found.
raise RuntimeError('Could not find any usable '
'configuration profiles')
return()

def parse_profile(self):
pass



View File

@ -15,6 +15,7 @@ import uuid
import validators import validators
import zlib import zlib
import lxml.etree import lxml.etree
from bs4 import BeautifulSoup
from collections import OrderedDict from collections import OrderedDict
from dns import resolver from dns import resolver
from email.utils import parseaddr as emailparse from email.utils import parseaddr as emailparse
@ -28,6 +29,7 @@ passlib_schemes = ['des_crypt', 'md5_crypt', 'sha256_crypt', 'sha512_crypt']
# Build various hash digest name lists # Build various hash digest name lists
digest_schemes = list(hashlib.algorithms_available) digest_schemes = list(hashlib.algorithms_available)
# Provided by zlib # Provided by zlib
# TODO
digest_schemes.append('adler32') digest_schemes.append('adler32')
digest_schemes.append('crc32') digest_schemes.append('crc32')


@ -36,8 +38,6 @@ crypt_map = {'sha512': crypt.METHOD_SHA512,
'md5': crypt.METHOD_MD5, 'md5': crypt.METHOD_MD5,
'des': crypt.METHOD_CRYPT} 'des': crypt.METHOD_CRYPT}




class XPathFmt(string.Formatter): class XPathFmt(string.Formatter):
def get_field(self, field_name, args, kwargs): def get_field(self, field_name, args, kwargs):
vals = self.get_value(field_name, args, kwargs), field_name vals = self.get_value(field_name, args, kwargs), field_name
@ -74,6 +74,41 @@ class detect(object):
return(None) return(None)
return() return()


def password_hash_salt(self, salted_hash):
_hash_list = salted_hash.split('$')
salt = _hash_list[2]
return(salt)

def remote_files(self, url_base, ptrn = None, flags = []):
with urlopen(url_base) as u:
soup = BeautifulSoup(u.read(), 'lxml')
urls = []
if 'regex' in flags:
if not isinstance(ptrn, str):
raise ValueError('"ptrn" must be a regex pattern to match '
'against')
else:
ptrn = re.compile(ptrn)
for u in soup.find_all('a'):
if not u.has_attr('href'):
continue
if 'regex' in flags:
if not ptrn.search(u.attrs['href']):
continue
if u.has_attr('href'):
urls.append(u.attrs['href'])
if not urls:
return(None)
# We certainly can't intelligently parse the printed timestamp since it
# varies so much and that'd be a nightmare to get consistent...
# But we CAN sort by filename.
if 'latest' in flags:
urls = sorted(list(set(urls)))
urls = urls[-1]
else:
urls = urls[0]
return(urls)

def gpgkeyID_from_url(self, url): def gpgkeyID_from_url(self, url):
with urlopen(url) as u: with urlopen(url) as u:
data = u.read() data = u.read()
@ -144,6 +179,9 @@ class generate(object):
_salt = crypt.mksalt(algo) _salt = crypt.mksalt(algo)
else: else:
_salt = salt _salt = salt
if not password:
# Intentionally empty password.
return('')
return(crypt.crypt(password, _salt)) return(crypt.crypt(password, _salt))


def hashlib_names(self): def hashlib_names(self):
@ -154,7 +192,9 @@ class generate(object):
hashes.append(h) hashes.append(h)
return(hashes) return(hashes)


def salt(self, algo = 'sha512'): def salt(self, algo = None):
if not algo:
algo = 'sha512'
algo = crypt_map[algo] algo = crypt_map[algo]
return(crypt.mksalt(algo)) return(crypt.mksalt(algo))


@ -574,6 +614,66 @@ class transform(object):
url['full_url'] += '#{0}'.format('#'.join(_f)) url['full_url'] += '#{0}'.format('#'.join(_f))
return(url) return(url)


def user(self, user_elem):
_attribs = ('hashed', 'hash_algo', 'salt')
acct = {}
for a in _attribs:
acct[a] = None
if len(user_elem):
elem = user_elem[0]
for a in _attribs:
if a in elem.attrib:
acct[a] = self.xml2py(elem.attrib[a], attrib = True)
if acct['hashed']:
if not acct['hash_algo']:
_hash = detect().password_hash(elem.text)
if _hash:
acct['hash_algo'] = _hash
else:
raise ValueError(
'Invalid salted password hash: {0}'.format(
elem.text)
)
acct['salt_hash'] = elem.text
acct['passphrase'] = None
else:
if not acct['hash_algo']:
acct['hash_algo'] = 'sha512'
acct['passphrase'] = elem.text
_saltre = re.compile('^\s*(auto|none|)\s*$', re.IGNORECASE)
if acct['salt']:
if _saltre.search(acct['salt']):
_salt = generate.salt(acct['hash_algo'])
acct['salt'] = _salt
else:
if not acct['hashed']:
acct['salt_hash'] = generate().hash_password(
acct['passphrase'],
algo = crypt_map[acct['hash_algo']])
acct['salt'] = detect().password_hash_salt(acct['salt_hash'])
if 'salt_hash' not in acct:
acct['salt_hash'] = generate().hash_password(
acct['passphrase'],
salt = acct['salt'],
algo = crypt_map[acct['hash_algo']])
return(acct)

def xml2py(self, value, attrib = True):
yes = re.compile('^\s*(y(es)?|true|1)\s*$', re.IGNORECASE)
no = re.compile('^\s*(no?|false|0)\s*$', re.IGNORECASE)
if no.search(value):
if attrib:
return(False)
else:
return(None)
elif yes.search(value):
# We handle the False case above.
return(True)
elif value.strip() == '':
return(None)
else:
return(value)

class valid(object): class valid(object):
def __init__(self): def __init__(self):
pass pass
@ -705,22 +805,32 @@ class valid(object):


class xml_supplicant(object): class xml_supplicant(object):
def __init__(self, cfg, profile = None, max_recurse = 5): def __init__(self, cfg, profile = None, max_recurse = 5):
raw = self._detect_cfg(cfg) self.selector_ids = ('id', 'name', 'uuid')
xmlroot = lxml.etree.fromstring(raw)
self.btags = {'xpath': {}, self.btags = {'xpath': {},
'regex': {}, 'regex': {},
'variable': {}} 'variable': {}}
raw = self._detect_cfg(cfg)
# This is changed in just a moment.
self.profile = profile
# This is retained so we can "refresh" the profile if needed.
self.orig_profile = profile
try:
self.xml = lxml.etree.fromstring(raw)
except lxml.etree.XMLSyntaxError:
raise ValueError('The configuration provided does not seem to be '
'valid')
self.get_profile(profile = profile)
self.xml = lxml.etree.fromstring(raw)
self.fmt = XPathFmt() self.fmt = XPathFmt()
self.max_recurse = max_recurse self.max_recurse = int(self.profile.xpath(
'//meta/max_recurse/text()')[0])
# I don't have permission to credit them, but to the person who helped # I don't have permission to credit them, but to the person who helped
# me with this regex - thank you. You know who you are. # me with this regex - thank you. You know who you are.
# Originally this pattern was the one from:
# https://stackoverflow.com/a/12728199/733214
self.ptrn = re.compile(('(?<=(?<!\{)\{)(?:[^{}]+' self.ptrn = re.compile(('(?<=(?<!\{)\{)(?:[^{}]+'
'|{{[^{}]*}})*(?=\}(?!\}))')) '|{{[^{}]*}})*(?=\}(?!\}))'))
self.root = lxml.etree.ElementTree(xmlroot) self.root = lxml.etree.ElementTree(self.xml)
if not profile:
self.profile = xmlroot.xpath('/bdisk/profile[1]')[0]
else:
self.profile = xmlroot.xpath(profile)[0]
self._parse_regexes() self._parse_regexes()
self._parse_variables() self._parse_variables()
@ -754,7 +864,8 @@ class xml_supplicant(object):


def _parse_regexes(self): def _parse_regexes(self):
for regex in self.profile.xpath('//meta/regexes/pattern'): for regex in self.profile.xpath('//meta/regexes/pattern'):
self.btags['regex'][regex.attrib['id']] = re.compile(regex.text) _key = 'regex%{0}'.format(regex.attrib['id'])
self.btags['regex'][_key] = regex.text
return() return()


def _parse_variables(self): def _parse_variables(self):
@ -764,6 +875,85 @@ class xml_supplicant(object):
] = variable.text ] = variable.text
return() return()


def btags_to_dict(self, text_in):
d = {}
ptrn_id = self.ptrn.findall(text_in)
if len(ptrn_id) >= 1:
for item in ptrn_id:
try:
btag, expr = item.split('%', 1)
if btag not in self.btags:
continue
if item not in self.btags[btag]:
self.btags[btag][item] = None
#self.btags[btag][item] = expr # remove me?
if btag == 'xpath':
d[item] = (btag, expr)
elif btag == 'variable':
d[item] = (btag, self.btags['variable'][item])
except ValueError:
return(d)
return(d)

def get_profile(self, profile = None):
"""Get a configuration profile.

Get a configuration profile from the XML object and set that as a
profile object. If a profile is specified, attempt to find it. If not,
follow the default rules as specified in __init__.
"""
if profile:
# A profile identifier was provided
if isinstance(profile, str):
_profile_name = profile
profile = {}
for i in self.selector_ids:
profile[i] = None
profile['name'] = _profile_name
elif isinstance(profile, dict):
for k in self.selector_ids:
if k not in profile.keys():
profile[k] = None
else:
raise TypeError('profile must be a string (name of profile), '
'a dictionary, or None')
xpath = '/bdisk/profile{0}'.format(self.xpath_selector(profile))
self.profile = self.xml.xpath(xpath)
if len(self.profile) != 1:
raise ValueError('Could not determine a valid, unique '
'profile; please check your profile '
'specifier(s)')
else:
# We need the actual *profile*, not a list of matching
# profile(s).
self.profile = self.profile[0]
if not len(self.profile):
raise RuntimeError('Could not find the profile specified in '
'the given configuration')
else:
# We need to find the default.
profiles = []
for p in self.xml.xpath('/bdisk/profile'):
profiles.append(p)
# Look for one named "default" or "DEFAULT" etc.
for idx, value in enumerate([e.attrib['name'].lower() \
for e in profiles]):
if value == 'default':
#self.profile = copy.deepcopy(profiles[idx])
self.profile = profiles[idx]
break
# We couldn't find a profile with a default name. Try to grab the
# first profile.
if self.profile is None:
# Grab the first profile.
if profiles:
self.profile = profiles[0]
else:
# No profiles found.
raise RuntimeError('Could not find any usable '
'configuration profiles')
return()

def get_path(self, element): def get_path(self, element):
path = element path = element
try: try:
@ -815,33 +1005,10 @@ class xml_supplicant(object):
_dictmap = self.btags_to_dict(element.text) _dictmap = self.btags_to_dict(element.text)
return(element) return(element)


def xpath_selector(self, selectors, def xpath_selector(self, selectors):
selector_ids = ('id', 'name', 'uuid')):
# selectors is a dict of {attrib:value} # selectors is a dict of {attrib:value}
xpath = '' xpath = ''
for i in selectors.items(): for i in selectors.items():
if i[1] and i[0] in selector_ids: if i[1] and i[0] in self.selector_ids:
xpath += '[@{0}="{1}"]'.format(*i) xpath += '[@{0}="{1}"]'.format(*i)
return(xpath) return(xpath)

def btags_to_dict(self, text_in):
d = {}
ptrn_id = self.ptrn.findall(text_in)
if len(ptrn_id) >= 1:
for item in ptrn_id:
try:
btag, expr = item.split('%', 1)
if btag not in self.btags:
continue
if item not in self.btags[btag]:
self.btags[btag][item] = None
#self.btags[btag][item] = expr # remove me?
if btag == 'xpath':
d[item] = (btag, expr)
elif btag == 'variable':
d[item] = (btag, self.btags['variable'][item])
except ValueError:
return(d)
return(d)



View File

@ -9,7 +9,7 @@
but now with the neat benefits of XPath! Everything you could do in build.ini's and more. but now with the neat benefits of XPath! Everything you could do in build.ini's and more.
See https://www.w3schools.com/xml/xpath_syntax.asp See https://www.w3schools.com/xml/xpath_syntax.asp
If you need a literal curly brace, double them (e.g. for "{foo}", use "{{foo}}"), If you need a literal curly brace, double them (e.g. for "{foo}", use "{{foo}}"),
UNLESS it's in a {regex%...} placeholder/filter (as part of the expression). --> UNLESS it's in a <regexes><pattern> as part of the expression. Those are taken as literal strings. -->
<pname>{xpath%../name/text()}</pname> <pname>{xpath%../name/text()}</pname>
</names> </names>
<desc>A rescue/restore live environment.</desc> <desc>A rescue/restore live environment.</desc>
@ -24,7 +24,7 @@
<!-- If the maximum level is reached, the substitution will evaluate as blank. --> <!-- If the maximum level is reached, the substitution will evaluate as blank. -->
<max_recurse>5</max_recurse> <max_recurse>5</max_recurse>
<!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain <!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain
items. See the manual for more information. --> items. See the manual for more information. NO btags within the patterns is allowed. -->
<regexes> <regexes>
<pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern> <pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern>
<pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern> <pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern>
@ -139,10 +139,10 @@
blank passphrase for all operations. --> blank passphrase for all operations. -->
<gpg keyid="none" gnupghome="none" publish="no" prompt_passphrase="no"> <gpg keyid="none" gnupghome="none" publish="no" prompt_passphrase="no">
<!-- The below is only used if we are generating a key (i.e. keyid="none"). --> <!-- The below is only used if we are generating a key (i.e. keyid="none"). -->
<key type="rsa" keysize="4096" expire="0"> <key algo="rsa" keysize="4096" expire="0">
<name>{xpath%../../../../meta/dev/author/text()}</name> <name>{xpath%../../../meta/dev/author/text()}</name>
<email>{xpath%../../../../meta/dev/email/text()}</email> <email>{xpath%../../../meta/dev/email/text()}</email>
<comment>for {xpath%../../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../../meta/uri/text()} | {xpath%../../../../meta/desc/text()}</comment> <comment>for {xpath%../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../meta/uri/text()} | {xpath%../../../meta/desc/text()}</comment>
</key> </key>
</gpg> </gpg>
<sync> <sync>
@ -263,10 +263,10 @@
</client> </client>
</pki> </pki>
<gpg keyid="none" gnupghome="none" publish="no" prompt_passphrase="no"> <gpg keyid="none" gnupghome="none" publish="no" prompt_passphrase="no">
<key type="rsa" keysize="4096" expire="0"> <key algo="rsa" keysize="4096" expire="0">
<name>{xpath%../../../../meta/dev/author/text()}</name> <name>{xpath%../../../meta/dev/author/text()}</name>
<email>{xpath%../../../../meta/dev/email/text()}</email> <email>{xpath%../../../meta/dev/email/text()}</email>
<comment>for {xpath%../../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../../meta/uri/text()} | {xpath%../../../../meta/desc/text()}</comment> <comment>for {xpath%../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../meta/uri/text()} | {xpath%../../../meta/desc/text()}</comment>
</key> </key>
</gpg> </gpg>
<sync> <sync>

View File

@ -9,7 +9,7 @@
but now with the neat benefits of XPath! Everything you could do in build.ini's and more. but now with the neat benefits of XPath! Everything you could do in build.ini's and more.
See https://www.w3schools.com/xml/xpath_syntax.asp See https://www.w3schools.com/xml/xpath_syntax.asp
If you need a literal curly brace, double them (e.g. for "{foo}", use "{{foo}}"), If you need a literal curly brace, double them (e.g. for "{foo}", use "{{foo}}"),
UNLESS it's in a {regex%...} placeholder/filter (as part of the expression). --> UNLESS it's in a <regexes><pattern> as part of the expression. Those are taken as literal strings. -->
<pname>{xpath%../name/text()}</pname> <pname>{xpath%../name/text()}</pname>
</names> </names>
<desc>A rescue/restore live environment.</desc> <desc>A rescue/restore live environment.</desc>
@ -24,7 +24,7 @@
<!-- If the maximum level is reached, the substitution will evaluate as blank. --> <!-- If the maximum level is reached, the substitution will evaluate as blank. -->
<max_recurse>5</max_recurse> <max_recurse>5</max_recurse>
<!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain <!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain
items. See the manual for more information. --> items. See the manual for more information. NO btags within the patterns is allowed. -->
<regexes> <regexes>
<pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern> <pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern>
<pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern> <pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern>
@ -62,7 +62,7 @@
<rootpath>/iso/latest</rootpath> <rootpath>/iso/latest</rootpath>
<tarball flags="regex,latest">{regex%tarball_x86_64}</tarball> <tarball flags="regex,latest">{regex%tarball_x86_64}</tarball>
<checksum hash_algo="sha1" <checksum hash_algo="sha1"
flags="none">sha1sums.txt</checksum> explicit="no">sha1sums.txt</checksum>
<sig keys="7F2D434B9741E8AC" <sig keys="7F2D434B9741E8AC"
keyserver="hkp://pool.sks-keyservers.net" keyserver="hkp://pool.sks-keyservers.net"
flags="regex,latest">{regex%sig_x86_64}</sig> flags="regex,latest">{regex%sig_x86_64}</sig>
@ -157,9 +157,9 @@
prompt_passphrase="no"> prompt_passphrase="no">
<!-- The below is only used if we are generating a key (i.e. keyid="none"). --> <!-- The below is only used if we are generating a key (i.e. keyid="none"). -->
<key algo="rsa" keysize="4096" expire="0"> <key algo="rsa" keysize="4096" expire="0">
<name>{xpath%../../../../meta/dev/author/text()}</name> <name>{xpath%../../../meta/dev/author/text()}</name>
<email>{xpath%../../../../meta/dev/email/text()}</email> <email>{xpath%../../../meta/dev/email/text()}</email>
<comment>for {xpath%../../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../../meta/uri/text()} | {xpath%../../../../meta/desc/text()}</comment> <comment>for {xpath%../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../meta/uri/text()} | {xpath%../../../meta/desc/text()}</comment>
</key> </key>
</gpg> </gpg>
<sync> <sync>

View File

@ -8,3 +8,6 @@
- in faq/ISOBIG.adoc and the doc section it references, make sure we reference that the package lists are now in the environment plugin! - in faq/ISOBIG.adoc and the doc section it references, make sure we reference that the package lists are now in the environment plugin!


- change all references to build.ini to something like "BDisk configuration file" - change all references to build.ini to something like "BDisk configuration file"

- reminder: users can specify a local file source for <sources><source> items by using "file:///absolute/path/to/file"
-- todo: add http auth, ftp, ftps