From b2498ba98d9b544ff0e324295fb499ba21049dcc Mon Sep 17 00:00:00 2001 From: r00t Date: Fri, 18 May 2018 08:13:46 -0400 Subject: [PATCH] checking in some work --- TODO | 9 + bdisk/GPG.py | 3 + bdisk/SSL.py | 4 + bdisk/confgen.py | 169 ++++++++++- bdisk/utils.py | 470 +++++++++++++++++++----------- docs/examples/multi_profile.xml | 475 +++++++++++++++++-------------- docs/examples/regen_multi.py | 7 +- docs/examples/single_profile.xml | 76 +++-- 8 files changed, 789 insertions(+), 424 deletions(-) diff --git a/TODO b/TODO index 946c8a6..fbb2a43 100644 --- a/TODO +++ b/TODO @@ -9,6 +9,15 @@ - for docs, 3.x (as of 3.10) was 2.4M. - GUI? at least for generating config... +- SSL key gen: +import OpenSSL +k = OpenSSL.crypto.PKey() +k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096) +x = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, + k, + cipher = 'aes256', + passphrase = 'test') + - need to package: python-hashid (https://psypanda.github.io/hashID/, https://github.com/psypanda/hashID, diff --git a/bdisk/GPG.py b/bdisk/GPG.py index 6f22c16..a0852be 100644 --- a/bdisk/GPG.py +++ b/bdisk/GPG.py @@ -3,6 +3,9 @@ import os import psutil import gpg.errors +# http://files.au.adversary.org/crypto/GPGMEpythonHOWTOen.html +# https://www.gnupg.org/documentation/manuals/gpgme.pdf + class GPGHandler(object): def __init__(self, gnupg_homedir = None, key_id = None, keyservers = None): self.home = gnupg_homedir diff --git a/bdisk/SSL.py b/bdisk/SSL.py index df36961..a991649 100644 --- a/bdisk/SSL.py +++ b/bdisk/SSL.py @@ -1 +1,5 @@ import OpenSSL +# https://cryptography.io/en/latest/x509/reference/#cryptography.x509.CertificateBuilder.sign +# migrate old functions of bSSL to use cryptography +# but still waiting on their recpipes. +# https://cryptography.io/en/latest/x509/tutorial/ \ No newline at end of file diff --git a/bdisk/confgen.py b/bdisk/confgen.py index cdf0e06..46f2aab 100755 --- a/bdisk/confgen.py +++ b/bdisk/confgen.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3.6 +# Ironically enough, I think building a GUI for this would be *cleaner*. +# Go figure. + import confparse import crypt import getpass @@ -15,7 +18,7 @@ transform = utils.transform() valid = utils.valid() # TODO: convert the restarts for prompts to continue's instead of letting them -# continue on to the next prompt. +# continue on to the next prompt. def pass_prompt(user): # This isn't in utils.prompts() because we need to use an instance of @@ -147,6 +150,9 @@ class ConfGenerator(object): self.get_accounts() self.get_sources() self.get_build() + self.get_iso() + self.get_ipxe() + self.get_pki() except KeyboardInterrupt: exit('\n\nCaught KeyboardInterrupt; quitting...') return() @@ -290,6 +296,8 @@ class ConfGenerator(object): '\nWhat is YOUR name?\nName: ')).strip() meta_items['dev']['email'] = (input('\nWhat is your email address?' '\nemail: ')).strip() + # TODO: this always returns invalid?? and doesn't seem to trigger + # the redo if not valid.email(meta_items['dev']['email']): print('Invalid; skipping...') meta_items['dev']['email'] = None @@ -395,7 +403,8 @@ class ConfGenerator(object): 'x86_64': ('(Also referred to by distros as ' '"64-bit")')} while more_sources: - if len(_arches) == len(_supported_arches): + # this doesn't trigger? maybe? + if len(_arches) >= len(_supported_arches): # All supported arches have been added. We currently don't # support mirror-balancing. TODO? print('\nCannot add more sources; all supported architectures ' @@ -403,7 +412,7 @@ class ConfGenerator(object): more_sources = False break if len(_arches) > 0: - print('\n(Currently added arches: {0})'.format( + print('\n\t(Currently added arches: {0})'.format( ', '.join(_arches))) _print_arches = '\n\t'.join( ['{0}:\t{1}'.format(*i) for i in _supported_arches.items()]) @@ -531,7 +540,7 @@ class ConfGenerator(object): continue sig.attrib['keys'] = sigkeys else: - sigkeys = detect.gpgkeyID_from_url(gpgsig) + sigkeys = detect.gpgkeyID_from_url(gpgsig['full_url']) if not isinstance(sigkeys, list): print('Could not properly parse any keys in the ' 'signature file. Restarting.') @@ -553,7 +562,8 @@ class ConfGenerator(object): print('\t\t{0}'.format(_uid['Name'])) for k in _uid: if k != 'Name': - print('\t\t\t{0}:\t{1}'.format(k, _uid[k])) + print('\t\t\t{0:<9} {1}'.format( + '{0}:'.format(k), _uid[k])) _key_chk = prompt.confirm_or_no(prompt = ( '\n{0} look correct?\n').format(_s)) if not _key_chk: @@ -633,22 +643,157 @@ class ConfGenerator(object): distro_path = self.profile.xpath('//paths/distros/text()')[0] except IndexError: distro_path = 'your "distros" path' - distro = (input('\nWhich distro plugin/distro base are you using? ' - 'See the manual for more information. A matching ' - 'plugin MUST exist in {0} for a build to ' - 'complete successfully! The default (Arch Linux, ' - '"archlinux") will be used if left blank.' - '\nDistro base: ').format( - distro_path)).strip() + distro = (input(('\nWhich distro plugin/distro base are you ' + 'using? See the manual for more information. A ' + 'matching plugin MUST exist in {0} for a build ' + 'to complete successfully! The default (Arch ' + 'Linux, "archlinux") will be used if left blank.' + '\nDistro base: ').format(distro_path))).strip() if distro == '': distro = 'archlinux' if not valid.plugin_name(distro): print('That is not a valid name. See the manual for examples ' 'and shipped plugins. Retrying.') continue + else: + has_distro = True distro_elem = lxml.etree.SubElement(build, 'distro') distro_elem.text = distro return() + + def get_iso(self): + print('\n++ ISO ++') + # We don't need to ask if it's multiarch if we only have one arch. + iso = lxml.etree.Element('iso') + _arches = [] + for _source in self.profile.xpath('sources/source'): + _arches.append(_source.attrib['arch']) + if len(_arches) < 2: + iso.attrib['multi_arch'] = _arches[0] + self.profile.append(iso) + # We have more than one arch, so we need to ask how they want to handle + # it. + _ma_strings = {'yes': ('a multi-arch ISO (both architectures on one ' + 'ISO)'), + 'no': ('separate image files for ' + '{0}').format(' and '.join(_arches))} + for a in _arches: + _ma_strings[a] = 'only build an image file for {0}'.format(a) + if len(_arches) > 1: + _multi_arch_input = None + while not _multi_arch_input: + print('\n++ ISO || MULTI-ARCH ++') + _multi_arch = (input(( + '\nYou have defined mutliple architecture sources. BDisk ' + 'allows you to build an ISO image (USB image, etc.) that ' + 'will support both architectures using the same file. ' + 'Please consult the manual if you need further ' + 'information.\nPossible values:\n' + '\n\t{0}\n\nMulti-arch: ').format( + '\n\t'.join( + ['{0}:\t{1}'.format(k, v) for k, v in _ma_strings.items()] + )))).strip().lower() + if _multi_arch not in _ma_strings.keys(): + print('Invalid selection; retrying.') + continue + else: + _multi_arch_input = _multi_arch + iso.attrib['multi_arch'] = _multi_arch_input + _gpg_sign = None + while not _gpg_sign: + print('\n++ ISO || SIGNING ++') + _gpg_input = prompt.confirm_or_no(prompt = ( + '\nWe can sign ISO image files using GPG (we\'ll give the ' + 'option to configure it a bit later).\nWould you like to sign ' + 'the ISO/USB image files with GPG?\n'), usage = ( + '{0} for yes, {1} for no...\n')) + _gpg_sign = ('yes' if _gpg_input else 'no') + iso.attrib['sign'] = _gpg_sign + self.profile.append(iso) + return() + + def get_ipxe(self): + print('\n++ iPXE ++') + ipxe = lxml.etree.Element('ipxe') + _ipxe = None + while not _ipxe: + _ipxe = prompt.confirm_or_no(prompt = ( + '\nBDisk has built-in support for iPXE (https://ipxe.org/, ' + 'see the manual for more information). Would you like to ' + 'build iPXE support?\n'), usage = ( + '{0} for yes, {1} for no...\n')) + _ipxe = ('yes' if _ipxe else 'no') + if _ipxe == 'yes': + print('\n++ iPXE || MINI-ISO ++') + _iso = prompt.confirm_or_no(prompt = ( + '\nWould you like to build a "mini-ISO" (see the manual) for ' + 'bootstrapping iPXE booting from USB or optical media?\n'), + usage = ('{0} for yes, {1} for no...\n')) + ipxe.attrib['iso'] = ('yes' if _iso else 'no') + print('\n++ iPXE || SIGNING ++') + _sign = prompt.confirm_or_no(prompt = ( + '\nBDisk can sign the mini-ISO and other relevant files for ' + 'iPXE builds using GPG. Would you like to sign the iPXE build ' + 'distributables? (You\'ll have the chance to configure GPG ' + 'later).\n'), usage = ('{0} for yes, {1} for no...\n')) + ipxe.attrib['sign'] = ('yes' if _sign else 'no') + _uri = None + while not _uri: + print('\n++ iPXE || URL ++') + _uri = (input( + '\niPXE uses a remote URI to boot. What URI should this ' + 'profile\'s iPXE use? (Consult the manual for more ' + 'information.)\niPXE Boot URL: ')).strip() + if not valid.url(_uri): + print('Invalid URL, retrying...') + _uri = None + continue + else: + uri = lxml.etree.SubElement(ipxe, 'uri') + uri.text = _uri + if _ipxe == 'yes': + self.profile.append(ipxe) + return() + + def get_pki(self): + print('\n++ SSL/TLS PKI ++') + pki = lxml.etree.Element('pki') + _pki = None + while not _pki: + _pki = prompt.confirm_or_no(prompt = ( + '\nWould you like to support SSL/TLS transport for various ' + 'functions? Currently this is only used for iPXE, but future ' + 'applications may be possible.\n'), + usage = ('{0} for yes, {1} for no...\n')) + if _pki: + _pki_url_chk = self.profile.xpath('ipxe/uri/text()') + _pki_url = (_pki_url_chk[0] if _pki_url_chk else None) + print('\n++ SSL/TLS PKI || OVERWRITE ++') + _overwrite = prompt.confirm_or_no(prompt = ( + '\nYou\'ll have the opportunity in a moment to configure ' + 'paths for the various files, but do you want BDisk to ' + 're-generate (and thus overwrite) any of the files it finds? ' + 'If you use these files for anything OTHER than BDisk (or ' + 'wish to keep persistent keys and certs), you should ' + 'DEFINITELY answer no here.\n'), + usage = ('{0} for yes, {1} for no...\n')) + pki.attrib['overwrite'] = ('yes' if _overwrite else 'no') + for x in ('ca', 'client'): + print('\n++ SSL/TLS PKI || {0} ++'.format(x.upper())) + _x = None + while not _x: + _x = prompt.ssl_object(x, _pki_url) + elem = lxml.etree.SubElement(pki, x) + _elems = {} + for e in _x['paths']: + _elems[e] = lxml.etree.SubElement(elem, e) + _elems[e].text = _x['paths'][e] + for e in _x['attribs']: + for a in _x['attribs'][e]: + _elems[e].attrib[a] = _x['attribs'][e][a] + if _pki: + self.profile.append(pki) + return() def main(): cg = ConfGenerator() diff --git a/bdisk/utils.py b/bdisk/utils.py index aaffd36..45d3da3 100644 --- a/bdisk/utils.py +++ b/bdisk/utils.py @@ -3,11 +3,11 @@ import crypt import GPG import hashid import hashlib +import iso3166 import os import pprint import re import string -import textwrap import uuid import validators import zlib @@ -17,6 +17,7 @@ from dns import resolver from email.utils import parseaddr as emailparse from passlib.context import CryptContext as cryptctx from urllib.parse import urlparse +from urllib.request import urlopen # Supported by all versions of GNU/Linux shadow passlib_schemes = ['des_crypt', 'md5_crypt', 'sha256_crypt', 'sha512_crypt'] @@ -26,19 +27,21 @@ digest_schemes = list(hashlib.algorithms_available) # Provided by zlib digest_schemes.append('adler32') digest_schemes.append('crc32') -#clean_digest_schemes = sorted(list(set(digest_schemes))) crypt_map = {'sha512': crypt.METHOD_SHA512, 'sha256': crypt.METHOD_SHA256, 'md5': crypt.METHOD_MD5, 'des': crypt.METHOD_CRYPT} -class XPathFmt(string.Formatter): - def __init__(self): - print('foo') +# These are *key* ciphers, for encrypting exported keys. +openssl_ciphers = ['aes128', 'aes192', 'aes256', 'bf', 'blowfish', + 'camellia128', 'camellia192', 'camellia256', 'cast', 'des', + 'des3', 'idea', 'rc2', 'seed'] +openssl_digests = ['blake2b512', 'blake2s256', 'gost', 'md4', 'md5', 'mdc2', + 'rmd160', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'] +class XPathFmt(string.Formatter): def get_field(self, field_name, args, kwargs): - # custom arg to specify if it's a regex pattern or not vals = self.get_value(field_name, args, kwargs), field_name if not vals[0]: vals = ('{{{0}}}'.format(vals[1]), vals[1]) @@ -51,7 +54,7 @@ class detect(object): def any_hash(self, hash_str): h = hashid.HashID() hashes = [] - for i in h.IdentifyHash(hash_str): + for i in h.identifyHash(hash_str): if i.extended: continue x = i.name @@ -74,7 +77,7 @@ class detect(object): return() def gpgkeyID_from_url(self, url): - with urlparse(url) as u: + with urlopen(url) as u: data = u.read() g = GPG.GPGHandler() key_ids = g.get_sigs(data) @@ -85,7 +88,7 @@ class detect(object): def _get_key(): key = None try: - key = g.get_key(keyID, secret = secret) + key = g.ctx.get_key(keyID, secret = secret) except GPG.gpg.errors.KeyNotFound: return(None) except Exception: @@ -93,16 +96,16 @@ class detect(object): return(key) uids = {} g = GPG.GPGHandler() - _orig_kl_mode = g.get_keylist_mode() + _orig_kl_mode = g.ctx.get_keylist_mode() if _orig_kl_mode != GPG.gpg.constants.KEYLIST_MODE_EXTERN: _key = _get_key() if not _key: - g.set_keylist_mode(GPG.gpg.constants.KEYLIST_MODE_EXTERN) + g.ctx.set_keylist_mode(GPG.gpg.constants.KEYLIST_MODE_EXTERN) _key = _get_key() else: _key = _get_key() if not _key: - g.set_keylist_mode(_orig_kl_mode) + g.ctx.set_keylist_mode(_orig_kl_mode) del(g) return(None) else: @@ -119,7 +122,7 @@ class detect(object): _u['Invalid'] = (True if _uid.invalid else False) _u['Revoked'] = (True if _uid.revoked else False) uids['User IDs'].append(_u) - g.set_keylist_mode(_orig_kl_mode) + g.ctx.set_keylist_mode(_orig_kl_mode) del(g) return(uids) @@ -163,7 +166,7 @@ class prompts(object): def confirm_or_no(self, prompt = '', invert = False, usage = '{0} to confirm, otherwise {1}...\n'): - # A simplified version of multiline_input, really. + # A simplified version of multiline_input(), really. # By default, Enter confirms (and returns True) and CTRL-d returns # False unless - you guessed it - invert is True. # usage is a string appended to prompt that explains which keys to use. @@ -224,11 +227,135 @@ class prompts(object): print(end_str) return('\n'.join(_lines)) - def path(self, path_desc): + def path(self, path_desc, empty_passthru = False): path = input(('\nWhere would you like to put {0}?\n' 'Path: ').format(path_desc)) + if empty_passthru: + if path.strip() == '': + return('') path = transform().full_path(path) return(path) + + def ssl_object(self, pki_role, cn_url): + ssl_vals = {'paths': {}, + 'attribs': {}, + 'subject': {}} + # pki_role should be 'ca' or 'client' + if pki_role not in ('ca', 'client'): + raise ValueError('pki_role must be either "ca" or "client"') + _attribs = {'cert': {'hash_algo': {'text': ('What hashing algorithm ' + 'do you want to use? (Default is sha512.)'), + 'prompt': 'Hashing algorithm: ', + 'options': openssl_digests, + 'default': 'sha512'}}, + 'key': {'cipher': {'text': ('What encryption algorithm/' + 'cipher do you want to use? (Default is ' + 'aes256.)'), + 'prompt': 'Cipher: ', + 'options': openssl_ciphers, + 'default': 'aes256'}, + # This can actually theoretically be anywhere from + # 512 to... who knows how high. I couldn't find the + # upper bound. So we just set it to sensible + # defaults. If they want something higher, they can + # edit the XML when they're done. + 'keysize': {'text': ('What keysize/length (in ' + 'bits) do you want the key to be? (Default is ' + '4096; much higher values are possible but ' + 'are untested and thus not supported by this ' + 'tool; feel free to edit the generated ' + 'configuration by hand.)'), + 'prompt': 'Keysize: ', + 'options': ['1024', '2048', '4096'], + 'default': '4096'}}} + _paths = {'cert': '(or read from) the certificate', + 'key': '(or read from) the key', + 'csr': ('(or read from) the certificate signing request (if ' + 'blank, we won\'t write to disk - the operation ' + 'will occur entirely in memory assuming we need to ' + 'generate/sign)')} + if pki_role == 'ca': + _paths['index'] = ('(or read from) the CA DB index file (if left ' + 'blank, one will not be used)') + _paths['serial'] = ('(or read from) the CA DB serial file (if ' + 'left blank, one will not be used)') + for a in _attribs: + ssl_vals['attribs'][a] = {} + for x in _attribs[a]: + ssl_vals['attribs'][a][x] = None + for p in _paths: + if p == 'csr': + _allow_empty = True + else: + _allow_empty = False + ssl_vals['paths'][p] = self.path(_paths[p], + empty_passthru = _allow_empty) + print() + if ssl_vals['paths'][p] == '': + ssl_vals['paths'][p] = None + if p in _attribs: + for x in _attribs[p]: + while not ssl_vals['attribs'][p][x]: + ssl_vals['attribs'][p][x] = (input( + ('\n{0}\n\n\t{1}\n\n{2}').format( + _attribs[p][x]['text'], + '\n\t'.join(_attribs[p][x]['options']), + _attribs[p][x]['prompt']) + )).strip().lower() + if ssl_vals['attribs'][p][x] not in \ + _attribs[p][x]['options']: + print(('\nInvalid selection; setting default ' + '({0}).').format(_attribs[p][x]['default'])) + ssl_vals['attribs'][p][x] = \ + _attribs[p][x]['default'] + _subject = {'countryName': {'text': ('the 2-letter country ' + 'abbreviation (must conform to ' + 'ISO3166 ALPHA-2)?\nCountry ' + 'code: '), + 'check': 'func', + 'func': valid().country_abbrev}, + 'localityName': {'text': ('the city/town/borough/locality ' + 'name?\nLocality: '), + 'check': None}, + 'stateOrProvinceName': {'text': ('the state/region ' + 'name (full string)?' + '\nRegion: '), + 'check': None}, + 'organization': {'text': ('your organization\'s name?' + '\nOrganization: '), + 'check': None}, + 'organizationalUnitName': {'text': ('your department/role/' + 'team/department name?' + '\nOrganizational ' + 'Unit: '), + 'check': None}, + 'emailAddress': {'text': ('the email address to be ' + 'associated with this ' + 'certificate/PKI object?\n' + 'Email: '), + 'check': 'func', + 'func': valid().email}} + for s in _subject: + ssl_vals['subject'][s] = None + for s in _subject: + while not ssl_vals['subject'][s]: + _input = (input( + ('\nWhat is {0}').format(_subject[s]['text']) + )).strip() + _chk = _subject[s]['check'] + if _chk: + if _chk == 'func': + _chk = _subject[s]['func'](_input) + if not _chk: + print('Invalid value; retrying.') + continue + print() + ssl_vals['subject'][s] = _input + _url = transform().url_to_dict(cn_url, no_None = True) + ssl_vals['subject']['commonName'] = _url['host'] + if pki_role == 'client': + ssl_vals['subject']['commonName'] += ' (Client)' + return(ssl_vals) class transform(object): def __init__(self): @@ -280,6 +407,7 @@ class transform(object): text_out = re.sub('[^\w]', '', text_out) return(text_out) + # noinspection PyDictCreation def url_to_dict(self, orig_url, no_None = False): def _getuserinfo(uinfo_str): if len(uinfo_str) == 0: @@ -332,25 +460,25 @@ class transform(object): return(None) else: return('') - params = {} + _params = {} for i in in_str.split(split_char): p = [x.strip() for x in i.split('=')] - params[p[0]] = p[1] - if not params: + _params[p[0]] = p[1] + if not _params: if not no_None: return(None) else: return('') - if not params and not no_None: + if not _params and not no_None: return(None) - return(params) + return(_params) _dflt_ports = _getdfltport() scheme = None - _scheme_re = re.compile('^([\w+\.-]+)(://.*)', re.IGNORECASE) + _scheme_re = re.compile('^([\w+.-]+)(://.*)', re.IGNORECASE) if not _scheme_re.search(orig_url): # They probably didn't prefix a URI signifier (RFC3986 § 3.1). # We'll add one for them. - url = 'http://' + url + url = 'http://' + orig_url scheme = 'http' else: # urlparse's .scheme? Total trash. @@ -407,7 +535,7 @@ class transform(object): 'url': orig_url} url['full_url'] = '{0}://'.format(scheme) if userinfo not in (None, ''): - url['full_url'] += '{user}:{password}@'.format(userinfo) + url['full_url'] += '{user}:{password}@'.format(**userinfo) url['full_url'] += host if port not in (None, ''): url['full_url'] += ':{0}'.format(port) @@ -424,145 +552,15 @@ class transform(object): url['full_url'] += '#{0}'.format('#'.join(_f)) return(url) -class xml_supplicant(object): - def __init__(self, cfg, profile = None, max_recurse = 5): - raw = self._detect_cfg(cfg) - xmlroot = lxml.etree.fromstring(raw) - self.btags = {'xpath': {}, - 'regex': {}} - self.fmt = XPathFmt() - self.max_recurse = max_recurse - #self.ptrn = re.compile('(?<=(?= self.max_recurse: - return(element) - if isinstance(element, lxml.etree._Element): - if isinstance(element, lxml.etree._Comment): - return(element) -# if len(element) == 0: -# print(element.text) - if element.text: - _dictmap = self.xpath_to_dict(element.text) - while _dictmap: - for elem in _dictmap: - if isinstance(_dictmap[elem], str): - try: - newpath = element.xpath(_dictmap[elem]) - except (AttributeError, IndexError, TypeError): - newpath = element - except lxml.etree.XPathEvalError: - return(element) - try: - self.btags['xpath'][elem] = self.substitute( - newpath, (recurse_count + 1))[0] - except (IndexError, TypeError): - raise ValueError( - ('Encountered an error while trying to ' - 'substitute {0} at {1}').format( - elem, self.get_path(element) - )) - print(element.text) - element.text = self.fmt.format( - element.text, - {**self.btags['xpath'], - **self.btags['regex']}) -# element.text = self.fmt.vformat( -# element.text, -# [], -# {**self.btags['xpath'], -# **self.btags['regex']}) -# element.text = (element.text).format( -# {**self.btags['xpath'], -# **self.btags['regex']}) - _dictmap = self.xpath_to_dict(element.text) - return(element) - - def xpath_selector(self, selectors, - selector_ids = ('id', 'name', 'uuid')): - # selectors is a dict of {attrib:value} - xpath = '' - for i in selectors.items(): - if i[1] and i[0] in selector_ids: - xpath += '[@{0}="{1}"]'.format(*i) - return(xpath) - - def xpath_to_dict(self, text_in): - d = None - ptrn_id = self.ptrn.findall(text_in) - if len(ptrn_id) >= 1: - for item in ptrn_id: - if not isinstance(d, dict): - d = {} - try: - _, xpath_expr = item.split('%', 1) - if _ not in self.btags: - continue - if item not in self.btags[_]: - self.btags[_][item] = None - if _ == 'regex': - _re = re.sub('^regex%', '', item) - _re = re.sub('{{(.*)}}', '\g<1>', _re) - # We use a native python object - self.btags['regex'][item] = re.compile(_re) - d[item] = xpath_expr - except ValueError: - return(None) - return(d) - - class valid(object): def __init__(self): pass + def country_abbrev(self, country_code): + if country_code not in iso3166.countries_by_alpha2: + return(False) + return(True) + def dns(self, addr): pass @@ -572,7 +570,7 @@ class valid(object): def email(self, addr): return( - isinstance(validators.email(emailparse(addr)[1]), + not isinstance(validators.email(emailparse(addr)[1]), validators.utils.ValidationFailure)) def gpgkeyID(self, key_id): @@ -626,9 +624,10 @@ class valid(object): def salt_hash(self, salthash): _idents = ''.join([i.ident for i in crypt_map if i.ident]) - _regex = re.compile('^(\$[{0}]\$)?[./0-9A-Za-z]{0,16}\$?'.format( + # noinspection PyStringFormat + _regex = re.compile('^(\$[{0}]\$)?[./0-9A-Za-z]{{0,16}}\$?'.format( _idents)) - if not regex.search(salthash): + if not _regex.search(salthash): return(False) return(True) @@ -650,7 +649,7 @@ class valid(object): return(True) def url(self, url): - if not re.search('^[\w+\.-]+://', url): + if not re.search('^[\w+.-]+://', url): # They probably didn't prefix a URI signifier (RFC3986 § 3.1). # We'll add one for them. url = 'http://' + url @@ -670,9 +669,152 @@ class valid(object): def uuid(self, uuid_str): is_uuid = True try: - u = uuid.UUID(uuid_in) + u = uuid.UUID(uuid_str) except ValueError: return(False) - if not uuid_in == str(u): + if not uuid_str == str(u): return(False) return(is_uuid) + +class xml_supplicant(object): + def __init__(self, cfg, profile = None, max_recurse = 5): + raw = self._detect_cfg(cfg) + xmlroot = lxml.etree.fromstring(raw) + self.btags = {'xpath': {}, + 'regex': {}, + 'variable': {}} + self.fmt = XPathFmt() + self.max_recurse = max_recurse + # I don't have permission to credit them, but to the person who helped + # me with this regex - thank you. You know who you are. + self.ptrn = re.compile(('(?<=(?= self.max_recurse: + return(element) + if isinstance(element, lxml.etree._Element): + if element.tag == 'regex': + return(element) + if isinstance(element, lxml.etree._Comment): + return(element) + if element.text: + _dictmap = self.btags_to_dict(element.text) + for elem in _dictmap: + # This is needed because _dictmap gets replaced below + if not _dictmap: + return(element) + _btag, _value = _dictmap[elem] + if isinstance(_value, str): + if _btag == 'xpath': + try: + newpath = element.xpath(_dictmap[elem][1]) + except (AttributeError, IndexError, TypeError): + newpath = element + except lxml.etree.XPathEvalError: + return(element) + try: + self.btags['xpath'][elem] = self.substitute( + newpath, (recurse_count + 1))[0] + except (IndexError, TypeError): + raise ValueError( + ('Encountered an error while trying to ' + 'substitute {0} at {1}').format( + elem, self.get_path(element) + )) + element.text = self.fmt.vformat( + element.text, + [], + {**self.btags['xpath'], + **self.btags['variable']}) + _dictmap = self.btags_to_dict(element.text) + return(element) + + def xpath_selector(self, selectors, + selector_ids = ('id', 'name', 'uuid')): + # selectors is a dict of {attrib:value} + xpath = '' + for i in selectors.items(): + if i[1] and i[0] in selector_ids: + xpath += '[@{0}="{1}"]'.format(*i) + return(xpath) + + def btags_to_dict(self, text_in): + d = {} + ptrn_id = self.ptrn.findall(text_in) + if len(ptrn_id) >= 1: + for item in ptrn_id: + try: + btag, expr = item.split('%', 1) + if btag not in self.btags: + continue + if item not in self.btags[btag]: + self.btags[btag][item] = None + #self.btags[btag][item] = expr # remove me? + if btag == 'xpath': + d[item] = (btag, expr) + elif btag == 'variable': + d[item] = (btag, self.btags['variable'][item]) + except ValueError: + return(d) + return(d) + + diff --git a/docs/examples/multi_profile.xml b/docs/examples/multi_profile.xml index 9a77387..c8325f6 100644 --- a/docs/examples/multi_profile.xml +++ b/docs/examples/multi_profile.xml @@ -1,233 +1,266 @@ - - - - BDisk - bdisk - - {xpath%../name/text()} - - A rescue/restore live environment. - - A. Dev Eloper - dev@domain.tld - https://domain.tld/~dev - - https://domain.tld/projname - 1.0.0 - - - 5 - - - - $6$7KfIdtHTcXwVrZAC$LZGNeMNz7v5o/cYuA48FAxtZynpIwO5B1CPGXnOW5kCTVpXVt4SypRqfM.AoKkFt/O7MZZ8ySXJmxpELKmdlF1 - - {xpath%//meta/names/uxname/text()} - - - {xpath%//meta/dev/author/text()} - testpassword - - - testuser - Test User - anothertestpassword - - - - - http://archlinux.mirror.domain.tld - /iso/latest - {xpath%../mirror/text()}{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz} - {xpath%../mirror/text()}{xpath%../webroot/text()}/sha1sums.txt - {xpath%../tarball/text()}.sig - - - http://archlinux32.mirror.domain.tld - /iso/latest - {xpath%../mirror/text()}/{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz} - cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e - {xpath%../tarball/text()}.sig - - - - - /var/tmp/{xpath%//meta/names/uxname/text()} - /var/tmp/chroots/{xpath%//meta/names/uxname/text()} - {xpath%../cache/text()}/overlay - ~/{xpath%//meta/names/uxname/text()}/templates - /mnt/{xpath%//meta/names/uxname/text()} - ~/{xpath%//meta/names/uxname/text()}/distros - ~/{xpath%//meta/names/uxname/text()}/results - {xpath%../dest/text()}/iso - {xpath%../dest/text()}/http - {xpath%../dest/text()}/tftp - {xpath%../dest/text()}/pki - - archlinux - - - - {xpath%//meta/dev/website/text()}/ipxe - - - - - {xpath%../../../build/paths/pki/text()}/ca.crt - + + 5 + + + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz\.sig$ + + + + /var/tmp/BDisk + + + + + $6$7KfIdtHTcXwVrZAC$LZGNeMNz7v5o/cYuA48FAxtZynpIwO5B1CPGXnOW5kCTVpXVt4SypRqfM.AoKkFt/O7MZZ8ySXJmxpELKmdlF1 + + {xpath%//meta/names/uxname/text()} + + + {xpath%//meta/dev/author/text()} + testpassword + + + testuser + Test User + anothertestpassword + + + + + http://archlinux.mirror.domain.tld + /iso/latest + {regex%tarball_x86_64} + sha1sums.txt + {regex%sig_x86_64} + + + http://archlinux32.mirror.domain.tld + /iso/latest + {regex%tarball_i686} + cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e + {regex%sig_i686} + + + + + {variable%bdisk_root}/cache + {variable%bdisk_root}/chroots + {variable%bdisk_root}/overlay + {variable%bdisk_root}/templates + /mnt/{xpath%//meta/names/uxname/text()} + {variable%bdisk_root}/distros + {variable%bdisk_root}/results + {variable%bdisk_root}/iso_overlay + {variable%bdisk_root}/http + {variable%bdisk_root}/tftp + {variable%bdisk_root}/pki + + archlinux + + + + {xpath%//meta/dev/website/text()}/ipxe + + + + + {xpath%../../../build/paths/pki/text()}/ca.crt + - - {xpath%../../../build/paths/pki/text()}/ca.key - - domain.tld - XX - Some City - Some State - Some Org, Inc. - Department Name - {xpath%../../../../meta/dev/email/text()} - - - - {xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt - - {xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key - - some client name - XX - Some City - Some State - Some Org, Inc. - Department Name - {xpath%../../../../meta/dev/email/text()} - - - - - - /srv/http/{xpath%../../meta/names/uxname/text()} - /tftproot/{xpath%../../meta/names/uxname/text()} - /srv/http/isos/{xpath%../../meta/names/uxname/text()} - - root - mirror.domain.tld - 22 - ~/.ssh/id_ed25519 - - - - - - - AnotherCD - bdisk_alt - {xpath%../name/text()} - - Another rescue/restore live environment. - - Another Dev Eloper - {xpath%//profile[@name="default"]/meta/dev/email/text()} - {xpath%//profile[@name="default"]/meta/dev/website/text()} - - https://domain.tld/projname - 0.0.1 - 5 - - - atotallyinsecurepassword - - testuser - Test User - atestpassword - - - - - http://archlinux.mirror.domain.tld - /iso/latest - {xpath%../mirror/text()}{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz} - {xpath%../mirror/text()}{xpath%../webroot/text()}/sha1sums.txt - {xpath%../tarball/text()}.sig - - - http://archlinux32.mirror.domain.tld - /iso/latest - {xpath%../mirror/text()}/{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz} - cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e - {xpath%../tarball/text()}.sig - - - - - /var/tmp/{xpath%//meta/names/uxname/text()} - /var/tmp/chroots/{xpath%//meta/names/uxname/text()} - {xpath%../cache/text()}/overlay - ~/{xpath%//meta/names/uxname/text()}/templates - /mnt/{xpath%//meta/names/uxname/text()} - ~/{xpath%//meta/names/uxname/text()}/distros - ~/{xpath%//meta/names/uxname/text()}/results - {xpath%../dest/text()}/iso - {xpath%../dest/text()}/http - {xpath%../dest/text()}/tftp - {xpath%../dest/text()}/pki - - archlinux - - - - {xpath%//meta/dev/website/text()}/ipxe - - - - {xpath%../../../build/paths/pki/text()}/ca.crt - - {xpath%../../../build/paths/pki/text()}/ca.key - - domain.tld - XX - Some City - Some State - Some Org, Inc. - Department Name - {xpath%../../../../meta/dev/email/text()} - - - - {xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt - - {xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key - - some client name - XX - Some City - Some State - Some Org, Inc. - Department Name - {xpath%../../../../meta/dev/email/text()} - - - - - - /srv/http/{xpath%../../meta/names/uxname/text()} - /tftproot/{xpath%../../meta/names/uxname/text()} - /srv/http/isos/{xpath%../../meta/names/uxname/text()} - - root - mirror.domain.tld - 22 - ~/.ssh/id_ed25519 - - - + + + + + + {xpath%../../../build/paths/pki/text()}/index.txt + {xpath%../../../build/paths/pki/text()}/serial + + {xpath%../../../build/paths/pki/text()}/ca.key + + domain.tld + XX + Some City + Some State + Some Org, Inc. + Department Name + {xpath%../../../../meta/dev/email/text()} + + + + {xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt + + {xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key + + some client name + XX + Some City + Some State + Some Org, Inc. + Department Name + {xpath%../../../../meta/dev/email/text()} + + + + + + /srv/http/{xpath%../../meta/names/uxname/text()} + /tftproot/{xpath%../../meta/names/uxname/text()} + /srv/http/isos/{xpath%../../meta/names/uxname/text()} + + root + mirror.domain.tld + 22 + ~/.ssh/id_ed25519 + + + + + + + AnotherCD + bdisk_alt + {xpath%../name/text()} + + Another rescue/restore live environment. + + Another Dev Eloper + + {xpath%//profile[@name="default"]/meta/dev/email/text()} + {xpath%//profile[@name="default"]/meta/dev/website/text()} + + https://domain.tld/projname + 0.0.1 + 5 + + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz\.sig$ + + + /var/tmp/BDisk + + + + atotallyinsecurepassword + + testuser + Test User + atestpassword + + + + + http://archlinux.mirror.domain.tld + /iso/latest + {regex%tarball_x86_64} + sha1sums.txt + {regex%sig_x86_64} + + + http://archlinux32.mirror.domain.tld + /iso/latest + {regex%tarball_i686} + cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e + {regex%sig_i686} + + + + + {variable%bdisk_root}/cache + {variable%bdisk_root}/chroots + {variable%bdisk_root}/overlay + {variable%bdisk_root}/templates + /mnt/{xpath%//meta/names/uxname/text()} + {variable%bdisk_root}/distros + {variable%bdisk_root}/results + {variable%bdisk_root}/iso_overlay + {variable%bdisk_root}/http + {variable%bdisk_root}/tftp + {variable%bdisk_root}/pki + + archlinux + + + + {xpath%//meta/dev/website/text()}/ipxe + + + + {xpath%../../../build/paths/pki/text()}/ca.crt + + {xpath%../../../build/paths/pki/text()}/index.txt + {xpath%../../../build/paths/pki/text()}/serial + {xpath%../../../build/paths/pki/text()}/ca.key + + domain.tld + XX + Some City + Some State + Some Org, Inc. + Department Name + {xpath%../../../../meta/dev/email/text()} + + + + {xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt + + {xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key + + some client name + XX + Some City + Some State + Some Org, Inc. + Department Name + {xpath%../../../../meta/dev/email/text()} + + + + + + /srv/http/{xpath%../../meta/names/uxname/text()} + /tftproot/{xpath%../../meta/names/uxname/text()} + /srv/http/isos/{xpath%../../meta/names/uxname/text()} + + root + mirror.domain.tld + 22 + ~/.ssh/id_ed25519 + + + diff --git a/docs/examples/regen_multi.py b/docs/examples/regen_multi.py index e2673ca..2f92570 100755 --- a/docs/examples/regen_multi.py +++ b/docs/examples/regen_multi.py @@ -3,8 +3,10 @@ import copy from lxml import etree +parser = etree.XMLParser(remove_blank_text = True) + with open('single_profile.xml', 'rb') as f: - xml = etree.fromstring(f.read()) + xml = etree.fromstring(f.read(), parser) single_profile = xml.xpath('/bdisk/profile[1]')[0] alt_profile = copy.deepcopy(single_profile) @@ -45,6 +47,9 @@ for e in accounts.iter(): e.attrib['sudo'] = 'no' # Delete the second user accounts.remove(accounts[2]) +author = alt_profile.xpath('/profile/meta/dev/author')[0] +author.addnext(etree.Comment( + ' You can reference other profiles within the same configuration. ')) xml.append(alt_profile) with open('multi_profile.xml', 'wb') as f: diff --git a/docs/examples/single_profile.xml b/docs/examples/single_profile.xml index dee8e60..fb8c217 100644 --- a/docs/examples/single_profile.xml +++ b/docs/examples/single_profile.xml @@ -23,13 +23,25 @@ 5 + + + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz$ + archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz\.sig$ + + + + /var/tmp/BDisk + $6$7KfIdtHTcXwVrZAC$LZGNeMNz7v5o/cYuA48FAxtZynpIwO5B1CPGXnOW5kCTVpXVt4SypRqfM.AoKkFt/O7MZZ8ySXJmxpELKmdlF1 {xpath%//meta/names/uxname/text()} - + {xpath%//meta/dev/author/text()} http://archlinux.mirror.domain.tld - /iso/latest - {xpath%../mirror/text()}{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{{4}}\.[0-9]{{2}}\.[0-9]{{2}}-x86_64\.tar\.gz} - - {xpath%../mirror/text()}{xpath%../webroot/text()}/sha1sums.txt - /iso/latest + {regex%tarball_x86_64} + sha1sums.txt + {xpath%../tarball/text()}.sig + flags="regex,latest">{regex%sig_x86_64} http://archlinux32.mirror.domain.tld - /iso/latest - {xpath%../mirror/text()}/{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{{4}}\.[0-9]{{2}}\.[0-9]{{2}}-i686\.tar\.gz} - + /iso/latest + {regex%tarball_i686} cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e {xpath%../tarball/text()}.sig + keyserver="hkp://pool.sks-keyservers.net" + flags="regex,latest">{regex%sig_i686} - /var/tmp/{xpath%//meta/names/uxname/text()} - /var/tmp/chroots/{xpath%//meta/names/uxname/text()} - {xpath%../cache/text()}/overlay - ~/{xpath%//meta/names/uxname/text()}/templates + {variable%bdisk_root}/cache + {variable%bdisk_root}/chroots + {variable%bdisk_root}/overlay + {variable%bdisk_root}/templates /mnt/{xpath%//meta/names/uxname/text()} - ~/{xpath%//meta/names/uxname/text()}/distros - ~/{xpath%//meta/names/uxname/text()}/results - {xpath%../dest/text()}/iso - {xpath%../dest/text()}/http - {xpath%../dest/text()}/tftp - {xpath%../dest/text()}/pki + {variable%bdisk_root}/distros + {variable%bdisk_root}/results + {variable%bdisk_root}/iso_overlay + {variable%bdisk_root}/http + {variable%bdisk_root}/tftp + {variable%bdisk_root}/pki archlinux - + {xpath%//meta/dev/website/text()}/ipxe - {xpath%../../../build/paths/pki/text()}/ca.crt + {xpath%../../../build/paths/pki/text()}/ca.crt - {xpath%../../../build/paths/pki/text()}/ca.key + + + + + {xpath%../../../build/paths/pki/text()}/index.txt + {xpath%../../../build/paths/pki/text()}/serial + + {xpath%../../../build/paths/pki/text()}/ca.key domain.tld XX @@ -108,9 +130,11 @@ - {xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt + {xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt - {xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key + {xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key some client name XX