updating some scripts - fixes, mostly. conf_minify works WAY better now.

2018-10-18 14:13:34 -04:00 · 2018-10-18 14:13:34 -04:00 · 6f450ab68f
parent d84a98520a
commit 6f450ab68f
8 changed files with 140 additions and 41 deletions
--- a/aif/scripts/post/sshsecure.py
+++ b/aif/scripts/post/sshsecure.py
@ -4,13 +4,101 @@
 # TODO: check for cryptography module. if it exists, we can do this entirely pythonically
 #       without ever needing to use subprocess/ssh-keygen, i think!

+# Thanks to https://stackoverflow.com/a/39126754.
+
+# stdlib
 import datetime
 import glob
 import os
 import pwd
 import re
 import shutil
-import subprocess
+import subprocess  # REMOVE WHEN SWITCHING TO PURE PYTHON
+#### PREP FOR PURE PYTHON IMPLEMENTATION ####
+# # non-stdlib - testing and automatic install if necessary.
+# # TODO #
+# - cryptography module won't generate new-format "openssh-key-v1" keys.
+# - See https://github.com/pts/py_ssh_keygen_ed25519 for possible conversion to python 3
+# - https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key
+# - https://github.com/pyca/cryptography/issues/3509 and https://github.com/paramiko/paramiko/issues/1136
+# has_crypto = False
+# pure_py = False
+# has_pip = False
+# pipver = None
+# try:
+#     import cryptography
+#     has_crypto = True
+# except ImportError:
+#     # We'll try to install it. We set up the logic below.
+#     try:
+#         import pip
+#         has_pip = True
+#         # We'll use these to create a temporary lib path and remove it when done.
+#         import sys
+#         import tempfile
+#     except ImportError:
+#         # ABSOLUTE LAST fallback, if we got to THIS case, is to use subprocess.
+#         has_pip = False
+#         import subprocess
+#
+# # Try installing it then!
+# if not all((has_crypto, )):
+#     # venv only included after python 3.3.x. We fallback to subprocess if we can't do dis.
+#     if sys.hexversion >= 0x30300f0:
+#         has_ensurepip = False
+#         import venv
+#         if not has_pip and sys.hexversion >= 0x30400f0:
+#             import ensurepip
+#             has_ensurepip = True
+#         temppath = tempfile.mkdtemp('_VENV')
+#         v = venv.create(temppath)
+#         if has_ensurepip and not has_pip:
+#             # This SHOULD be unnecessary, but we want to try really hard.
+#             ensurepip.bootstrap(root = temppath)
+#             import pip
+#             has_pip = True
+#         if has_pip:
+#             pipver = pip.__version__.split('.')
+#             # A thousand people are yelling at me for this.
+#             if int(pipver[0]) >= 10:
+#                 from pip._internal import main as pipinstall
+#             else:
+#                 pipinstall = pip.main
+#             if int(pipver[0]) >= 8:
+#                 pipcmd = ['install',
+#                           '--prefix={0}'.format(temppath),
+#                           '--ignore-installed']
+#             else:
+#                 pipcmd = ['install',
+#                           '--install-option="--prefix={0}"'.format(temppath),
+#                           '--ignore-installed']
+#             # Get the lib path.
+#             libpath = os.path.join(temppath, 'lib')
+#             if os.path.exists('{0}64'.format(libpath)) and not os.path.islink('{0}64'.format(libpath)):
+#                 libpath += '64'
+#             for i in os.listdir(libpath):  # TODO: make this more sane. We cheat a bit here by making assumptions.
+#                 if re.search('python([0-9]+(\.[0-9]+)?)?$', i):
+#                     libpath = os.path.join(libpath, i)
+#                     break
+#             libpath = os.path.join(libpath, 'site-packages')
+#             sys.prefix = temppath
+#             for m in ('cryptography', 'ed25519'):
+#                 pipinstall(['install', 'cryptography'])
+#             sys.path.append(libpath)
+#             try:
+#                 import cryptography
+#                 has_crypto = True
+#             except ImportError:  # All that trouble for nothin'. Shucks.
+#                 pass
+#
+# if all((has_crypto, )):
+#     pure_py = True
+#
+# if pure_py:
+#     from cryptography.hazmat.primitives import serialization as crypto_serialization
+#     from cryptography.hazmat.primitives.asymmetric import rsa
+#     from cryptography.hazmat.backends import default_backend as crypto_default_backend
+#

 conf_options = {}
 conf_options['sshd'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256',
@ -31,12 +119,13 @@ conf_options['ssh'] = {'Host': {'*': {'KexAlgorithms': 'curve25519-sha256@libssh
                                      'PubkeyAuthentication': 'yes',
                                      'HostKeyAlgorithms': 'ssh-ed25519-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,ssh-ed25519,ssh-rsa'}}}
 # Uncomment below if Github still needs diffie-hellman-group-exchange-sha1 sometimes.
+# For what it's worth, it doesn't seem to.
 #conf_options['ssh']['Host']['github.com'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256,' +
 #                                             'diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1'}


 def hostKeys(buildmoduli):
-    # Starting haveged should help lessen the time load, but not much.
+    # Starting haveged should help lessen the time load a non-negligible amount, especially on virtual platforms.
    if os.path.lexists('/usr/bin/haveged'):
        # We could use psutil here, but then that's a python dependency we don't need.
        # We could parse the /proc directory, but that's quite unnecessary. pgrep's installed by default on Arch.
--- a/ref/ascii
+++ b/ref/ascii
@ -1,2 +1,4 @@
 http://plato.asu.edu/MAT420/beginning_perl/3145_AppF.pdf
-http://www.profdavis.net/ASCII_table.pdf
+http://www.profdavis.net/ASCII_table.pdf
+http://www.aboutmyip.com/AboutMyXApp/AsciiChart.jsp
+http://www.robelle.com/smugbook/ascii.html
--- a/ref/progress.py
+++ b/ref/progress.py
@ -11,7 +11,7 @@ def waiter(seconds = 1):
    max = len(anims) - 1
    global is_done
    print('Beginning dhparam gen...')
-    # This is just an example commant that takes a looong time.
+    # This is just an example command that takes a looong time.
    c = subprocess.Popen(['openssl', 'dhparam', '-out', '/tmp/dhpem', '4096'],
    #c = subprocess.Popen(['openssl', 'genrsa', '-out', '/tmp/dhpem', '4096'],
                         stdout = subprocess.PIPE,
--- a/ref/python.tips_tricks_and_dirty_hacks
+++ b/ref/python.tips_tricks_and_dirty_hacks
@ -26,7 +26,7 @@ ref: https://www.python.org/dev/peps/pep-0008/#imports

 ###############################################################################

-To programmatically install modules via pip if they aren't installed:
+To programmatically install modules via pip if they aren't installed (BROKEN IN RECENT PIP VERSIONS; pip._internal.main() i think):

 ____
 import importlib
@ -108,4 +108,13 @@ log = logger.log(name = 'project.name')
 ###############################################################################

 # TODO #
-https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module
+https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module
+
+
+###############################################################################
+
+To issue an equivalent of "reset" command in linux, assuming console is ANSI-compat,
+
+print('\x1bc', end = '')
+
+###############################################################################
--- a/ssl_tls/certparser.py
+++ b/ssl_tls/certparser.py
@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3

 # stdlib
 import argparse
--- a/storage/backups/borg/backup.py
+++ b/storage/backups/borg/backup.py
@ -45,6 +45,12 @@ class Backup(object):
        if self.args['oper'] == 'backup':
            for d in (self.args['mysqldir'], self.args['stagedir']):
                os.makedirs(d, exist_ok = True, mode = 0o700)
+        if self.args['oper'] == 'restore':
+            self.args['target_dir'] = os.path.abspath(os.path.expanduser(
+                                                    self.args['target_dir']))
+            os.makedirs(os.path.dirname(self.args['oper']),
+                        exist_ok = True,
+                        mode = 0o700)
        ### LOGGING ###
        # Thanks to:
        # https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
@ -255,7 +261,8 @@ class Backup(object):
                                                  self.cfg['config']['host'],
                                                  r,
                                                  self.args['archive']))
-            # TODO: support specific path of extract?
+            _cmd.append(os.path.abspath(self.args['target_dir']))
+            # TODO: support specific path inside archive?
            # if so, append path(s) here.
            _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
            self.logger.debug('VARS: {0}'.format(vars()))
--- a/storage/mtree_to_xml.py
+++ b/storage/mtree_to_xml.py
@ -159,6 +159,8 @@ class MtreeXML(object):
            return(out)
        def _unset_parse(unsetline):
            out = {}
+            if unsetline[1] == 'all':
+                return(copy.deepcopy(self._tplitem))
            for i in unsetline:
                out[i] = self._tplitem[i]
            return(out)
--- a/text/conf_minify.py
+++ b/text/conf_minify.py
@ -31,21 +31,15 @@ class ConfStripper(object):
        if not self.comments:
            if len(self.comment_syms) == 1:
                if self.inline:
-                    self.regexes.append(re.compile(
-                                                '^(.*){0}.*'.format(
-                                                        self.comment_syms[0])))
+                    self.regexes.append(re.compile('^([^{0}]*){0}.*'.format(self.comment_syms[0])))
                else:
-                    self.regexes.append(re.compile(
-                                                '^(\s*){0}.*'.format(
-                                                        self.comment_syms[0])))
+                    self.regexes.append(re.compile('^(\s*){0}.*'.format(self.comment_syms[0])))
            else:
                syms = '|'.join(self.comment_syms)
                if self.inline:
-                    self.regexes.append(re.compile(
-                                                '^(.*)({0}).*'.format(syms)))
+                    self.regexes.append(re.compile('^(.*)({0}).*'.format(syms)))
                else:
-                    self.regexes.append(re.compile(
-                                                '^(\s*)({0}).*'.format(syms)))
+                    self.regexes.append(re.compile( '^(\s*)({0}).*'.format(syms)))
        return()

    def parse(self, path):
@ -73,7 +67,7 @@ class ConfStripper(object):
            return(None)
        try:
            with open(path, 'r') as f:
-                conf = [i.strip() for i in f.readlines()]
+                conf = f.readlines()
        except UnicodeDecodeError:  # It's a binary file. Oops.
            if self.cli:
                print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path))
@ -89,8 +83,13 @@ class ConfStripper(object):
        # Okay, so now we can actually parse.
        # Comments first.
        for idx, line in enumerate(conf):
+            if line.strip() == '':
+                continue
            for r in self.regexes:
                conf[idx] = r.sub('\g<1>', conf[idx])
+            if conf[idx].strip() == '':  # The line was "deleted".
+                conf[idx] = None
+        conf = [i for i in conf if i is not None]
        # Then leading spaces...
        if not self.leading:
            for idx, line in enumerate(conf):
@ -101,7 +100,7 @@ class ConfStripper(object):
                conf[idx] = conf[idx].rstrip()
        # Lastly, if set, remove blank lines.
        if not self.whitespace:
-            conf = [i for i in conf if i != '']
+            conf = [i for i in conf if i.strip() != '']
        return(conf)

    def recurse(self, path):
@ -139,8 +138,7 @@ class ConfStripper(object):
                f.write(new_content)
        except PermissionError:
            if self.cli:
-                print('{0}: Cannot write (insufficient permission)'.format(
-                                                                        path))
+                print('{0}: Cannot write (insufficient permission)'.format(path))
            return()
        return()

@ -160,9 +158,7 @@ class ConfStripper(object):
        return(realpaths)

 def parseArgs():
-    args = argparse.ArgumentParser(description = ('Remove extraneous ' +
-                                                  'formatting/comments from ' +
-                                                  'files'))
+    args = argparse.ArgumentParser(description = ('Remove extraneous formatting/comments from files'))
    args.add_argument('-c', '--keep-comments',
                      dest = 'comments',
                      action = 'store_true',
@ -172,15 +168,13 @@ def parseArgs():
                      dest = 'comment_syms',
                      action = 'append',
                      default = [],
-                      help = ('The character(s) to be treated as comments. ' +
-                              'Can be specified multiple times (one symbol ' +
-                              'per flag, please, unless a specific sequence ' +
-                              'denotes a comment). Default is just #'))
+                      help = ('The character(s) to be treated as comments. '
+                              'Can be specified multiple times (one symbol per flag, please, unless a specific '
+                              'sequence denotes a comment). Default is just #'))
    args.add_argument('-i', '--no-inline',
                      dest = 'inline',
                      action = 'store_false',
-                      help = ('If specified, do NOT parse the files as ' +
-                              'having inline comments (the default is to ' +
+                      help = ('If specified, do NOT parse the files as having inline comments (the default is to '
                              'look for inline comments)'))
    args.add_argument('-s', '--keep-whitespace',
                      dest = 'whitespace',
@ -189,17 +183,15 @@ def parseArgs():
    args.add_argument('-t', '--keep-trailing',
                      dest = 'trailing',
                      action = 'store_true',
-                      help = ('If specified, retain trailing whitespace on ' +
-                              'lines'))
+                      help = ('If specified, retain trailing whitespace on lines'))
    args.add_argument('-l', '--no-leading-whitespace',
                      dest = 'leading',
                      action = 'store_false',
                      help = ('If specified, REMOVE leading whitespace'))
-    args.add_argument('-d', '--dry-run',
+    args.add_argument('-w', '--write',
                      dest = 'dry_run',
-                      action = 'store_true',
-                      help = ('If specified, don\'t actually overwrite the ' +
-                              'file(s) - just print to stdout instead'))
+                      action = 'store_false',
+                      help = ('If specified, overwrite the file(s) instead of just printing to stdout'))
    args.add_argument('-S', '--no-symlinks',
                      dest = 'symlinks',
                      action = 'store_false',
@ -207,10 +199,8 @@ def parseArgs():
    args.add_argument('paths',
                      metavar = 'PATH/TO/DIR/OR/FILE',
                      nargs = '+',
-                      help = ('The path(s) to the file(s) to strip down. If ' +
-                              'a directory is given, files will ' +
-                              'recursively be modified (unless -d/--dry-run ' +
-                              'is specified). Can be specified multiple ' +
+                      help = ('The path(s) to the file(s) to strip down. If a directory is given, files will '
+                              'recursively be printed (unless -w/--write is specified). Can be specified multiple '
                              'times'))
    return(args)