From 46400303731c09d8165ad7a8f4259dc04833ce39 Mon Sep 17 00:00:00 2001 From: brent s Date: Sat, 28 Apr 2018 07:59:30 -0400 Subject: [PATCH] adding logger lib and conf_minify --- lib/python/logger.py | 109 ++++++++++++ ref/python.tips_tricks_and_dirty_hacks | 25 +++ text/conf_minify.py | 225 +++++++++++++++++++++++++ 3 files changed, 359 insertions(+) create mode 100755 lib/python/logger.py create mode 100755 text/conf_minify.py diff --git a/lib/python/logger.py b/lib/python/logger.py new file mode 100755 index 0000000..54c2db6 --- /dev/null +++ b/lib/python/logger.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +# The logfile. +dflt_logfile = '/var/log/optools/optools.log' + +# The default log level. Can be one of (in increasing levels of output): +# critical +# error +# warning +# info +# debug +# "debug" may log sensitive information! Do *not* use it unless ABSOLUTELY +# NECESSARY. +dflt_loglevel = 'warning' + +# stdlib +import datetime +import logging +import logging.handlers +import os + +class log(object): + def __init__(self, loglvl = dflt_loglevel, logfile = dflt_logfile, + logname = 'optools'): + # Loglevel mappings. + self.loglvls = {'critical': logging.CRITICAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG} + self.loglvl = loglvl.lower() + if self.loglvl not in self.loglvls: + raise ValueError(('{0} is not one of: ' + + '{1}').format(loglvl, + ', '.join(self.loglvls.keys()))) + self.Logger = logging.getLogger(logname) + self.logfile = os.path.abspath(os.path.expanduser(logfile)) + try: + os.makedirs(os.path.dirname(self.logfile), + exist_ok = True, + mode = 0o700) + except Exception as e: + # Make this non-fatal since we also log to journal for systemd? + raise e + self.systemd() + self.journald() + self.Logger.setLevel(self.loglvls[self.loglvl]) + self.log_handlers() + + def systemd(self): + # Add journald support if we're on systemd. + # We probably are since we're most likely on Arch, but we don't want to + # make assumptions. + self.systemd = False + _sysd_chk = ['/run/systemd/system', + '/dev/.run/systemd', + '/dev/.systemd'] + for _ in _sysd_chk: + if os.path.isdir(_): + self.systemd = True + break + return() + + def journald(self): + if not self.systemd: + return() + try: + from systemd import journal + except ImportError: + try: + import pip + pip.main(['install', '--user', 'systemd']) + from systemd import journal + except Exception as e: + # Build failed. Missing gcc, disk too full, whatever. + self.systemd = False + return() + + def log_handlers(self): + # Log formats + if self.systemd: + _jrnlfmt = logging.Formatter(fmt = ('{levelname}: {message} ' + + '({filename}:{lineno})'), + style = '{', + datefmt = '%Y-%m-%d %H:%M:%S') + _logfmt = logging.Formatter(fmt = ('{asctime}:{levelname}: {message} (' + + '{filename}:{lineno})'), + style = '{', + datefmt = '%Y-%m-%d %H:%M:%S') + # Add handlers + _dflthandler = logging.handlers.RotatingFileHandler(self.logfile, + encoding = 'utf8', + # 1GB + maxBytes = 1073741824, + backupCount = 5) + _dflthandler.setFormatter(_logfmt) + _dflthandler.setLevel(self.loglvls[self.loglvl]) + if self.systemd: + from systemd import journal + try: + h = journal.JournaldLogHandler() + except AttributeError: # Uses the other version + h = journal.JournalHandler() + h.setFormatter(_jrnlfmt) + h.setLevel(self.loglvls[self.loglvl]) + self.Logger.addHandler(h) + self.Logger.addHandler(_dflthandler) + self.Logger.info('Logging initialized') + return() diff --git a/ref/python.tips_tricks_and_dirty_hacks b/ref/python.tips_tricks_and_dirty_hacks index e22a446..4a9a70f 100644 --- a/ref/python.tips_tricks_and_dirty_hacks +++ b/ref/python.tips_tricks_and_dirty_hacks @@ -82,5 +82,30 @@ class ClassName(object): for i in kwargs.keys(): setattr(self, i, kwargs[i]) ---- + ############################################################################### +To store stdout and stderr to different files in a subprocess call: +---- +with open('/tmp/test.o', 'w') as out, open('/tmp/test.e', 'w') as err: + subprocess.run(['command'], stdout = out, stderr = err) +---- +############################################################################### + +To use optools logging lib (or other "shared" modules): +---- +import os +import re +import importlib +spec = importlib.util.spec_from_file_location( + 'logger', + '/opt/dev/optools/lib/python/logger.py') +logger = importlib.util.module_from_spec(spec) +spec.loader.exec_module(logger) +log = logger.log(name = 'project.name') +---- + +############################################################################### + +# TODO # +https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module \ No newline at end of file diff --git a/text/conf_minify.py b/text/conf_minify.py new file mode 100755 index 0000000..607f54e --- /dev/null +++ b/text/conf_minify.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3.6 + +import argparse +import os +import re +import stat + +class ConfStripper(object): + def __init__(self, paths, comments = False, comment_syms = '#', + inline = True, whitespace = False, leading = True, + trailing = False, dry_run = False, symlinks = True): + if __name__ == '__main__': + # We're being run as a CLI utility, not an import. + self.cli = True + else: + self.cli = False + self.paths = self.paths_parser(paths) + self.comments = comments + self.comment_syms = comment_syms + self.inline = inline + self.whitespace = whitespace + self.leading = leading + self.trailing = trailing + self.dry_run = dry_run + self.symlinks = symlinks + self.prep() + + def prep(self): + self.regexes = [] + # In self.regexes, we group what we *keep* into group #1. + if not self.comments: + if len(self.comment_syms) == 1: + if self.inline: + self.regexes.append(re.compile( + '^(.*){0}.*'.format( + self.comment_syms[0]))) + else: + self.regexes.append(re.compile( + '^(\s*){0}.*'.format( + self.comment_syms[0]))) + else: + syms = '|'.join(self.comment_syms) + if self.inline: + self.regexes.append(re.compile( + '^(.*)({0}).*'.format(syms))) + else: + self.regexes.append(re.compile( + '^(\s*)({0}).*'.format(syms))) + return() + + def parse(self, path): + if os.path.islink(path): + if self.symlinks: + # Check for a broken symlink + try: + os.stat(path) + except FileNotFoundError: + if self.cli: + print('{0}: Broken symlink'.format(path)) + return(None) + else: + # We don't even WANT to follow symlinks. + if self.cli: + print('{0}: Symlink'.format(path)) + return(None) + if stat.S_ISSOCK(os.stat(path).st_mode): # It's a socket + if self.cli: + print('{0}: Socket'.format(path)) + return(None) + if stat.S_ISFIFO(os.stat(path).st_mode): # It's a named pipe + if self.cli: + print('{0}: Named pipe'.format(path)) + return(None) + try: + with open(path, 'r') as f: + conf = [i.strip() for i in f.readlines()] + except UnicodeDecodeError: # It's a binary file. Oops. + if self.cli: + print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path)) + return(None) + except PermissionError: + if self.cli: + print('{0}: Insufficient permission'.format(path)) + return(None) + except Exception as e: + if self.cli: + print('{0}: {1}'.format(path, e)) + return(None) + # Okay, so now we can actually parse. + # Comments first. + for idx, line in enumerate(conf): + for r in self.regexes: + conf[idx] = r.sub('\g<1>', conf[idx]) + # Then leading spaces... + if not self.leading: + for idx, line in enumerate(conf): + conf[idx] = conf[idx].lstrip() + # Then trailing spaces... + if not self.trailing: + for idx, line in enumerate(conf): + conf[idx] = conf[idx].rstrip() + # Lastly, if set, remove blank lines. + if not self.whitespace: + conf = [i for i in conf if i != ''] + return(conf) + + def recurse(self, path): + files = [] + for r, d, f in os.walk(path): + for i in f: + files.append(os.path.join(r, i)) + return(files) + + def main(self): + # Handle the files first. + for p in self.paths['files']: + try: + new_content = '\n'.join(self.parse(p)) + except TypeError: # Binary file, etc. + continue + self.writer(p, new_content) + # Then the directories... + for d in self.paths['dirs']: + for f in self.recurse(d): + try: + new_content = '\n'.join(self.parse(f)) + except TypeError: # Binary file, etc. + continue + self.writer(f, new_content) + return() + + def writer(self, path, new_content): + if self.dry_run: + print('\n== {0} =='.format(path)) + print(new_content, end = '\n\n') + return() + try: + with open(path, 'w') as f: + f.write(new_content) + except PermissionError: + if self.cli: + print('{0}: Cannot write (insufficient permission)'.format( + path)) + return() + return() + + def paths_parser(self, paths): + realpaths = {'files': [], + 'dirs': []} + for p in paths: + path = os.path.abspath(os.path.expanduser(p)) + if not os.path.exists(path): + if self.cli: + print('{0} does not exist; skipping...'.format(path)) + continue + if os.path.isfile(path): + realpaths['files'].append(path) + elif os.path.isdir(path): + realpaths['dirs'].append(path) + return(realpaths) + +def parseArgs(): + args = argparse.ArgumentParser(description = ('Remove extraneous ' + + 'formatting/comments from ' + + 'files')) + args.add_argument('-c', '--keep-comments', + dest = 'comments', + action = 'store_true', + help = ('If specified, retain all comments')) + args.add_argument('-C', '--comment-symbol', + metavar = 'SYMBOL', + dest = 'comment_syms', + action = 'append', + default = [], + help = ('The character(s) to be treated as comments. ' + + 'Can be specified multiple times (one symbol ' + + 'per flag, please, unless a specific sequence ' + + 'denotes a comment). Default is just #')) + args.add_argument('-i', '--no-inline', + dest = 'inline', + action = 'store_false', + help = ('If specified, do NOT parse the files as ' + + 'having inline comments (the default is to ' + + 'look for inline comments)')) + args.add_argument('-s', '--keep-whitespace', + dest = 'whitespace', + action = 'store_true', + help = ('If specified, retain whitespace')) + args.add_argument('-t', '--keep-trailing', + dest = 'trailing', + action = 'store_true', + help = ('If specified, retain trailing whitespace on ' + + 'lines')) + args.add_argument('-l', '--no-leading-whitespace', + dest = 'leading', + action = 'store_false', + help = ('If specified, REMOVE leading whitespace')) + args.add_argument('-d', '--dry-run', + dest = 'dry_run', + action = 'store_true', + help = ('If specified, don\'t actually overwrite the ' + + 'file(s) - just print to stdout instead')) + args.add_argument('-S', '--no-symlinks', + dest = 'symlinks', + action = 'store_false', + help = ('If specified, don\'t follow symlinks')) + args.add_argument('paths', + metavar = 'PATH/TO/DIR/OR/FILE', + nargs = '+', + help = ('The path(s) to the file(s) to strip down. If ' + + 'a directory is given, files will ' + + 'recursively be modified (unless -d/--dry-run ' + + 'is specified). Can be specified multiple ' + + 'times')) + return(args) + +def main(): + args = vars(parseArgs().parse_args()) + if not args['comment_syms']: + args['comment_syms'].append('#') + c = ConfStripper(**args) + c.main() + +if __name__ == '__main__': + main()