From 080ee26fffc4298eb8dc4d4b0e6531af5fed58fc Mon Sep 17 00:00:00 2001 From: brent s Date: Sat, 14 Apr 2018 13:03:07 -0400 Subject: [PATCH] updating some sksdump stuff, adding a filesizes summer --- gpg/sksdump.py | 30 +++++++---- storage/sum_filesizes.py | 108 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 11 deletions(-) create mode 100755 storage/sum_filesizes.py diff --git a/gpg/sksdump.py b/gpg/sksdump.py index 24263ba..1f23da5 100755 --- a/gpg/sksdump.py +++ b/gpg/sksdump.py @@ -31,7 +31,8 @@ def getDefaults(): 'sync': {'throttle': 0}, 'paths': {'basedir': '/var/lib/sks', 'destdir': '/srv/http/sks/dumps', - 'rsync': 'root@mirror.square-r00t.net:/srv/http/sks/dumps'}, + 'rsync': 'root@mirror.square-r00t.net:/srv/http/sks/dumps', + 'sksbin': '/usr/bin/sks'}, 'runtime': {'nodump': None, 'nocompress': None, 'nosync': None}} ## Build out the default .ini. dflt_str = ('# IMPORTANT: This script uses certain permissions functions that require some forethought.\n' + @@ -53,13 +54,13 @@ def getDefaults(): '# These services will be started/stopped, in order, before/after dumps. ' + 'Comma-separated.\nsvcs = {3}\n# The path to the logfile.\nlogfile = {4}\n# The number ' + 'of days of rotated key dumps. If None, don\'t rotate.\ndays = {5}\n# How many keys to include in each ' + - 'dump file.\ndumpkeys = {6}\n\n').format(d['user'], - d['group'], - d['compress'], - ','.join(d['svcs']), - d['logfile'], - d['days'], - d['dumpkeys']) + 'dump file.\ndumpkeys = {6}\n\n\n').format(d['user'], + d['group'], + d['compress'], + ','.join(d['svcs']), + d['logfile'], + d['days'], + d['dumpkeys']) # [sync] d = dflt['sync'] dflt_str += ('# This section controls sync settings.\n[sync]\n# This setting is what the speed should be throttled to, '+ @@ -69,9 +70,11 @@ def getDefaults(): dflt_str += ('# This section controls where stuff goes and where we should find it.\n[paths]\n# ' + 'Where your SKS DB is.\nbasedir = {0}\n# This is the base directory where the dumps should go.\n' + '# There will be a sub-directory created for each date.\ndestdir = {1}\n# The ' + - 'path for rsyncing the dumps. If None, don\'t rsync.\nrsync = {2}\n\n').format(d['basedir'], - d['destdir'], - d['rsync']) + 'path for rsyncing the dumps. If None, don\'t rsync.\nrsync = {2}\n' + + '# The path to the sks binary to use\nsksbin = {3}\n\n').format(d['basedir'], + d['destdir'], + d['rsync'], + d['sksbin']) # [runtime] d = dflt['runtime'] dflt_str += ('# This section controls runtime options. These can be overridden at the commandline.\n' + @@ -265,6 +268,11 @@ def parseArgs(): default = paths['basedir'], dest = 'basedir', help = 'The directory which holds your SKS DB.') + args.add_argument('-x', + '--sks-binary', + default = paths['sksbin'], + dest = 'sksbin', + help = 'The path to the SKS binary/executable to use to performt the dumps.') args.add_argument('-e', '--destdir', default = paths['destdir'], diff --git a/storage/sum_filesizes.py b/storage/sum_filesizes.py new file mode 100755 index 0000000..24a6c33 --- /dev/null +++ b/storage/sum_filesizes.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +# STDIN should be a list of "humanized" filesizes, e.g. 1.4G, 112K, etc. +# You should be able to get this with some careful grepping and awking. + +import re +import sys + +if sys.stdin.isatty(): + exit('You need to pipe in the file sizes you want summed.') + +class BitFmtr(str): + # https://bugs.python.org/msg123686 + def __format__(self, fmt): + if fmt[0] == 'u': + s = self.upper() + fmt = fmt[1:] + elif fmt[0] == 'l': + s = self.lower() + fmt = fmt[1:] + else: + s = str(self) + return(s.__format__(fmt)) + +class deHumanizer(object): + def __init__(self, lines_in): + # lines_in should be a list of values to sum in "human" format. + # Supports terabits/bytes down to single bit/bytes + self.bytes = 0 + self.bits = 0 + self.sizes = [i.strip() for i in lines_in] + _bytes = '^[0-9]+(\.[0-9]+)?\s*{0}B?$' + _bits = '^[0-9]+(\.[0-9]+)?\s*({0:l}b?|{0:u}b)$' + # Use a tuple. THe first item is the pattern to match, the second is + # the multiplier to get it to bytes. + # 1(TB) = 1099511627776 bytes/8796093022208 bits + self.terabyte = (re.compile(_bytes.format('T')), 1099511627776) + # 1 = 137438953472 bytes/1099511627776 bits + self.terabit = (re.compile(_bits.format(BitFmtr('T'))), 137438953472) + # 1 = 1073741824 bytes/8589934592 bits + self.gigabyte = (re.compile(_bytes.format('G')), 1073741824) + # 1 = 134217728 bytes/1073741824 bits + self.gigabit = (re.compile(_bits.format(BitFmtr('G'))), 134217728) + # 1 = 1048576 bytes/8388608 bits + self.megabyte = (re.compile(_bytes.format('M')), 1048576) + # 1 = 131072 bytes/1048576 bits + self.megabit = (re.compile(_bits.format(BitFmtr('M'))), 131072) + # 1 = 1024 bytes/8192 bits + self.kilobyte = (re.compile(_bytes.format('K')), 1024) + # 1 = 128 bytes/1024 bits + self.kilobit = (re.compile(_bits.format(BitFmtr('K'))), 128) + # 1 = 1 byte/8 bits + # We don't use the pre-built pattern for these because you don't ever + # see "## Bb" etc. Bytes are the default, IIRC, on Linux. + self.byte = (re.compile('^[0-9]+(\.[0-9]+)?\s*B?$'), 1) + # 1 = 0.125 bytes/1 bit + self.bit = (re.compile('^[0-9]+(\.[0-9]+)?\s*b$'), 0.125) + + def convert(self): + idx = 0 + for i in self.sizes[:]: + try: + _factor = float(re.sub('^([0-9\.]+)\s*.*$', '\g<1>', i)) + except ValueError: + print('{0} does not seem to be a size; skipping'.format(i)) + self.sizes[idx] = 0 # Null it out since we couldn't parse it. + # It's much more likely to be a smaller size than a larger one, + # statistically-speaking, and more likely to be in bytes than bits. + for r in (self.byte, self.kilobyte, self.megabyte, self.gigabyte, + self.terabyte, self.bit, self.kilobit, self.megabit, + self.gigabit, self.terabit): + if r[0].search(i): + self.sizes[idx] = float(_factor * r[1]) + idx += 1 + break +# # We didn't match, so remove it. +# self.sizes[idx] = 0 +# idx += 1 + return() + + def get_sums(self): + self.bytes = int(sum(self.sizes)) + self.bits = int(self.bytes * 8) + self.kilobytes = int(self.bytes / self.kilobyte[1]) + self.kilobits = int(self.bytes / self.kilobit[1]) + self.megabytes = int(self.bytes / self.megabyte[1]) + self.megabits = int(self.bytes / self.megabit[1]) + self.gigabytes = int(self.bytes / self.gigabyte[1]) + self.gigabits = int(self.bytes / self.gigabit[1]) + self.terabytes = int(self.bytes / self.terabyte[1]) + self.terabits = int(self.bytes / self.terabit[1]) + return() + +def main(data): + dh = deHumanizer(data) + dh.convert() + dh.get_sums() + print('TOTAL:') + print('GB: {0}'.format(dh.gigabytes)) + print('Gb: {0}'.format(dh.gigabits)) + print('MB: {0}'.format(dh.megabytes)) + print('Mb: {0}'.format(dh.megabits)) + print('B: {0}'.format(dh.bytes)) + print('b: {0}'.format(dh.bits)) + return() + +if __name__ == '__main__': + main([i.strip() for i in sys.stdin.readlines() if i not in ('', '\n')])