diff --git a/gpg/sksdump.py b/gpg/sksdump.py index 8d2e40b..59ef224 100755 --- a/gpg/sksdump.py +++ b/gpg/sksdump.py @@ -3,6 +3,7 @@ # Thanks to Matt Rude and https://gist.github.com/mattrude/b0ac735d07b0031bb002 so I can know what the hell I'm doing. import argparse +import base64 import configparser import datetime import getpass @@ -31,61 +32,75 @@ def getDefaults(): 'sync': {'throttle': 0}, 'paths': {'basedir': '/var/lib/sks', 'destdir': '/srv/http/sks/dumps', - 'rsync': 'root@mirror.square-r00t.net:/srv/http/sks/dumps', + 'rsync': ('root@mirror.square-r00t.net:' + + '/srv/http/sks/dumps'), 'sksbin': '/usr/bin/sks'}, 'runtime': {'nodump': None, 'nocompress': None, 'nosync': None}} ## Build out the default .ini. - dflt_str = ('# IMPORTANT: This script uses certain permissions functions that require some forethought.\n' + - '# You can either run as root, which is the "easy" way, OR you can run as the sks user.\n' + - '# Has to be one or the other; you\'ll SERIOUSLY mess things up otherwise.\n' + - '# If you run as the sks user, MAKE SURE the following is set in your sudoers\n' + - '# (where SKSUSER is the username sks runs as):\n#\tCmnd_Alias SKSCMDS = ' + - '/usr/bin/systemctl start sks-db,\\\n#\t\t/usr/bin/systemctl stop sks-db,\\\n#\t\t' + - '/usr/bin/systemctl start sks-recon,\\\n#\t\t/usr/bin/systemctl stop sks-recon\n#\t' + - 'SKSUSER ALL = NOPASSWD: SKSCMDS\n\n') - dflt_str += ('# This was written for systemd systems only. Tweaking would be needed for non-systemd systems\n' + - '# (since every non-systemd uses their own init system callables...)\n\n') - # [system] - d = dflt['system'] - dflt_str += ('## SKSDUMP CONFIG FILE ##\n\n# This section controls various system configuration.\n' + - '[system]\n# This should be the user SKS runs as.\nuser = {0}\n# This is the group that' + - 'SKS runs as.\ngroup = {1}\n# If None, don\'t compress dumps.\n# If one of: ' + - 'xz, gz, bz2, or lrz (for lrzip) then use that compression algo.\ncompress = {2}\n' + - '# These services will be started/stopped, in order, before/after dumps. ' + - 'Comma-separated.\nsvcs = {3}\n# The path to the logfile.\nlogfile = {4}\n# The number ' + - 'of days of rotated key dumps. If None, don\'t rotate.\ndays = {5}\n# How many keys to include in each ' + - 'dump file.\ndumpkeys = {6}\n\n\n').format(d['user'], - d['group'], - d['compress'], - ','.join(d['svcs']), - d['logfile'], - d['days'], - d['dumpkeys']) - # [sync] - d = dflt['sync'] - dflt_str += ('# This section controls sync settings.\n[sync]\n# This setting is what the speed should be throttled to, '+ - 'in KiB/s. Use 0 for no throttling.\nthrottle = {0}\n\n').format(d['throttle']) - # [paths] - d = dflt['paths'] - dflt_str += ('# This section controls where stuff goes and where we should find it.\n[paths]\n# ' + - 'Where your SKS DB is.\nbasedir = {0}\n# This is the base directory where the dumps should go.\n' + - '# There will be a sub-directory created for each date.\ndestdir = {1}\n# The ' + - 'path for rsyncing the dumps. If None, don\'t rsync.\nrsync = {2}\n' + - '# The path to the sks binary to use\nsksbin = {3}\n\n').format(d['basedir'], - d['destdir'], - d['rsync'], - d['sksbin']) - # [runtime] - d = dflt['runtime'] - dflt_str += ('# This section controls runtime options. These can be overridden at the commandline.\n' + - '# They take no values; they\'re merely options.\n[runtime]\n# Don\'t dump any keys.\n' + - '# Useful for dedicated in-transit/prep boxes.\n;nodump\n# Don\'t compress the dumps, even if ' + - 'we have a compression scheme specified in [system:compress].\n;nocompress\n# Don\'t sync to' + - 'another server/path, even if one is specified in [paths:rsync].\n;nosync\n') + dflt_b64 = ("""IyBJTVBPUlRBTlQ6IFRoaXMgc2NyaXB0IHVzZXMgY2VydGFpbiBwZXJtaXNz + aW9ucyBmdW5jdGlvbnMgdGhhdCByZXF1aXJlIHNvbWUKIyBmb3JldGhvdWdo + dC4KIyBZb3UgY2FuIGVpdGhlciBydW4gYXMgcm9vdCwgd2hpY2ggaXMgdGhl + ICJlYXN5IiB3YXksIE9SIHlvdSBjYW4gcnVuIGFzIHRoZQojIHNrcyB1c2Vy + IChvci4uLiB3aGF0ZXZlciB1c2VyIHlvdXIgU0tTIGluc3RhbmNlIHJ1bnMg + YXMpLgojIEl0IGhhcyB0byBiZSBvbmUgb3IgdGhlIG90aGVyOyB5b3UnbGwg + U0VSSU9VU0xZIG1lc3MgdGhpbmdzIHVwIG90aGVyd2lzZS4KIyBJZiB5b3Ug + cnVuIGFzIHRoZSBza3MgdXNlciwgTUFLRSBTVVJFIHRoZSBmb2xsb3dpbmcg + aXMgc2V0IGluIHlvdXIgc3Vkb2VycwojICh3aGVyZSBTS1NVU0VSIGlzIHRo + ZSB1c2VybmFtZSBza3MgcnVucyBhcyk6CiMJQ21uZF9BbGlhcyBTS1NDTURT + ID0gL3Vzci9iaW4vc3lzdGVtY3RsIHN0YXJ0IHNrcy1kYixcCiMJICAgICAg + ICAgICAgICAgICAgICAgL3Vzci9iaW4vc3lzdGVtY3RsIHN0b3Agc2tzLWRi + LFwKIyAgICAgICAgICAgICAgICAgICAgICAgIC91c3IvYmluL3N5c3RlbWN0 + bCBzdGFydCBza3MtcmVjb24sXAojCQkgICAgICAgICAgICAgICAgIC91c3Iv + YmluL3N5c3RlbWN0bCBzdG9wIHNrcy1yZWNvbgojCVNLU1VTRVIgQUxMID0g + Tk9QQVNTV0Q6IFNLU0NNRFMKCiMgVGhpcyB3YXMgd3JpdHRlbiBmb3Igc3lz + dGVtZCBzeXN0ZW1zIG9ubHkuIFR3ZWFraW5nIHdvdWxkIGJlIG5lZWRlZCBm + b3IKIyBub24tc3lzdGVtZCBzeXN0ZW1zIChzaW5jZSBldmVyeSBub24tc3lz + dGVtZCB1c2VzIHRoZWlyIG93biBpbml0IHN5c3RlbQojIGNhbGxhYmxlcy4u + LikKCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMj + IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMKCiMgVGhp + cyBzZWN0aW9uIGNvbnRyb2xzIHZhcmlvdXMgc3lzdGVtIGNvbmZpZ3VyYXRp + b24uCltzeXN0ZW1dCgojIFRoaXMgc2hvdWxkIGJlIHRoZSB1c2VyIFNLUyBy + dW5zIGFzLgp1c2VyID0gc2tzCgojIFRoaXMgaXMgdGhlIGdyb3VwIHRoYXQg + U0tTIHJ1bnMgYXMuCmdyb3VwID0gc2tzCgojIElmIGVtcHR5LCBkb24ndCBj + b21wcmVzcyBkdW1wcy4KIyBJZiBvbmUgb2Y6IHh6LCBneiwgYnoyLCBvciBs + cnogKGZvciBscnppcCkgdGhlbiB1c2UgdGhhdCBjb21wcmVzc2lvbiBhbGdv + LgojIE5vdGUgdGhhdCBscnppcCByZXF1aXJlcyBleHRyYSBpbnN0YWxsYXRp + b24uCmNvbXByZXNzID0geHoKCiMgVGhlc2Ugc2VydmljZXMgd2lsbCBiZSBz + dG9wcGVkL3N0YXJ0ZWQsIGluIG9yZGVyLCBiZWZvcmUvYWZ0ZXIgZHVtcHMu + IElmIG1vcmUKIyB0aGFuIG9uZSwgc2VwZXJhdGUgYnkgY29tbWFzLgpzdmNz + ID0gc2tzLWRiLHNrcy1yZWNvbgoKIyBUaGUgcGF0aCB0byB0aGUgbG9nZmls + ZS4KbG9nZmlsZSA9IC92YXIvbG9nL3Nrc2R1bXAubG9nCgojIFRoZSBudW1i + ZXIgb2YgZGF5cyBvZiByb3RhdGVkIGtleSBkdW1wcy4gSWYgZW1wdHksIGRv + bid0IHJvdGF0ZS4KZGF5cyA9IDEKCiMgSG93IG1hbnkga2V5cyB0byBpbmNs + dWRlIGluIGVhY2ggZHVtcCBmaWxlLgpkdW1wa2V5cyA9IDE1MDAwCgoKIyBU + aGlzIHNlY3Rpb24gY29udHJvbHMgc3luYyBzZXR0aW5ncy4KW3N5bmNdCgoj + IFRoaXMgc2V0dGluZyBpcyB3aGF0IHRoZSBzcGVlZCBzaG91bGQgYmUgdGhy + b3R0bGVkIHRvLCBpbiBLaUIvcy4gSWYgZW1wdHkgb3IKIyAwLCBwZXJmb3Jt + IG5vIHRocm90dGxpbmcuCnRocm90dGxlID0gMAoKCiMgVGhpcyBzZWN0aW9u + IGNvbnRyb2xzIHdoZXJlIHN0dWZmIGdvZXMgYW5kIHdoZXJlIHdlIHNob3Vs + ZCBmaW5kIGl0LgpbcGF0aHNdCgojIFdoZXJlIHlvdXIgU0tTIERCIGlzLgpi + YXNlZGlyID0gL3Zhci9saWIvc2tzCgojIFRoaXMgaXMgdGhlIGJhc2UgZGly + ZWN0b3J5IHdoZXJlIHRoZSBkdW1wcyBzaG91bGQgZ28uCiMgVGhlcmUgd2ls + bCBiZSBhIHN1Yi1kaXJlY3RvcnkgY3JlYXRlZCBmb3IgZWFjaCBkYXRlLgpk + ZXN0ZGlyID0gL3Nydi9odHRwL3Nrcy9kdW1wcwoKIyBUaGUgcGF0aCBmb3Ig + cnN5bmNpbmcgdGhlIGR1bXBzLiBJZiBlbXB0eSwgZG9uJ3QgcnN5bmMuCnJz + eW5jID0gcm9vdEBtaXJyb3Iuc3F1YXJlLXIwMHQubmV0Oi9zcnYvaHR0cC9z + a3MvZHVtcHMKCiMgVGhlIHBhdGggdG8gdGhlIHNrcyBiaW5hcnkgdG8gdXNl + Lgpza3NiaW4gPSAvdXNyL2Jpbi9za3MKCgojIFRoaXMgc2VjdGlvbiBjb250 + cm9scyBydW50aW1lIG9wdGlvbnMuIFRoZXNlIGNhbiBiZSBvdmVycmlkZGVu + IGF0IHRoZQojIGNvbW1hbmRsaW5lLiBUaGV5IHRha2Ugbm8gdmFsdWVzOyB0 + aGV5J3JlIG1lcmVseSBvcHRpb25zLgpbcnVudGltZV0KCiMgRG9uJ3QgZHVt + cCBhbnkga2V5cy4KIyBVc2VmdWwgZm9yIGRlZGljYXRlZCBpbi10cmFuc2l0 + L3ByZXAgYm94ZXMuCjtub2R1bXAKCiMgRG9uJ3QgY29tcHJlc3MgdGhlIGR1 + bXBzLCBldmVuIGlmIHdlIGhhdmUgYSBjb21wcmVzc2lvbiBzY2hlbWUgc3Bl + Y2lmaWVkIGluCiMgdGhlIFtzeXN0ZW06Y29tcHJlc3NdIHNlY3Rpb246ZGly + ZWN0aXZlLgo7bm9jb21wcmVzcwoKIyBEb24ndCBzeW5jIHRvIGFub3RoZXIg + c2VydmVyL3BhdGgsIGV2ZW4gaWYgb25lIGlzIHNwZWNpZmllZCBpbiBbcGF0 + aHM6cnN5bmNdLgo7bm9zeW5j""") realcfg = configparser.ConfigParser(defaults = dflt, allow_no_value = True) if not os.path.isfile(cfgfile): with open(cfgfile, 'w') as f: - f.write(dflt_str) + f.write(base64.b64decode(dflt_b64).decode('utf-8')) realcfg.read(cfgfile) return(realcfg) @@ -135,7 +150,9 @@ def destPrep(args): if getpass.getuser() == 'root': uid = getpwnam(args['user']).pw_uid gid = getgrnam(args['group']).gr_gid - for d in (args['destdir'], nowdir): # we COULD set it as part of the os.makedirs, but iirc it doesn't set it for existing dirs + # we COULD set it as part of the os.makedirs, but iirc it doesn't set + # it for existing dirs. + for d in (args['destdir'], nowdir): os.chown(d, uid, gid) if os.path.isdir(curdir): os.remove(curdir) @@ -168,7 +185,9 @@ def compressDB(args): if not args['compress']: return() curdir = os.path.join(args['destdir'], NOWstr) - for thisdir, dirs, files in os.walk(curdir): # I use os.walk here because we might handle this differently in the future... + # I use os.walk here because we might handle this differently in the + # future... + for thisdir, dirs, files in os.walk(curdir): files.sort() for f in files: fullpath = os.path.join(thisdir, f) @@ -177,22 +196,30 @@ def compressDB(args): # However, I can't do this on memory-constrained systems for lrzip. # See: https://github.com/kata198/python-lrzip/issues/1 with open(args['logfile'], 'a') as f: - f.write('===== {0} Now compressing {1} =====\n'.format(str(datetime.datetime.utcnow()), fullpath)) + f.write('===== {0} Now compressing {1} =====\n'.format( + str(datetime.datetime.utcnow()), + fullpath)) if args['compress'].lower() == 'gz': import gzip - with open(fullpath, 'rb') as fh_in, gzip.open(newfile, 'wb') as fh_out: + with open(fullpath, 'rb') as fh_in, gzip.open(newfile, + 'wb') as fh_out: fh_out.writelines(fh_in) elif args['compress'].lower() == 'xz': import lzma - with open(fullpath, 'rb') as fh_in, lzma.open(newfile, 'wb', preset = 9|lzma.PRESET_EXTREME) as fh_out: + with open(fullpath, 'rb') as fh_in, \ + lzma.open(newfile, + 'wb', + preset = 9|lzma.PRESET_EXTREME) as fh_out: fh_out.writelines(fh_in) elif args['compress'].lower() == 'bz2': import bz2 - with open(fullpath, 'rb') as fh_in, bz2.open(newfile, 'wb') as fh_out: + with open(fullpath, 'rb') as fh_in, bz2.open(newfile, + 'wb') as fh_out: fh_out.writelines(fh_in) elif args['compress'].lower() == 'lrz': import lrzip - with open(fullpath, 'rb') as fh_in, open(newfile, 'wb') as fh_out: + with open(fullpath, 'rb') as fh_in, open(newfile, + 'wb') as fh_out: fh_out.write(lrzip.compress(fh_in.read())) os.remove(fullpath) if getpass.getuser() == 'root': @@ -212,7 +239,8 @@ def syncDB(args): if args['throttle'] > 0.0: cmd.insert(-1, '--bwlimit={0}'.format(str(args['throttle']))) with open(args['logfile'], 'a') as f: - f.write('===== {0} Rsyncing to mirror =====\n'.format(str(datetime.datetime.utcnow()))) + f.write('===== {0} Rsyncing to mirror =====\n'.format( + str(datetime.datetime.utcnow()))) with open(args['logfile'], 'a') as f: subprocess.run(cmd, stdout = f, stderr = f) return() @@ -223,8 +251,10 @@ def parseArgs(): paths = cfg['paths'] sync = cfg['sync'] runtime = cfg['runtime'] - args = argparse.ArgumentParser(description = 'sksdump - a tool for dumping the SKS Database', - epilog = 'brent s. || 2017 || https://square-r00t.net') + args = argparse.ArgumentParser(description = ('sksdump - a tool for ' + + 'dumping an SKS Database'), + epilog = ('brent s. || 2018 || ' + + 'https://square-r00t.net')) args.add_argument('-u', '--user', default = system['user'], @@ -245,7 +275,9 @@ def parseArgs(): '--services', default = system['svcs'], dest = 'svcs', - help = 'A comma-separated list of services that will be stopped/started for the dump (in the provided order).') + help = ('A comma-separated list of services that will ' + + 'be stopped/started for the dump (in the ' + + 'provided order).')) args.add_argument('-l', '--log', default = system['logfile'], @@ -272,23 +304,28 @@ def parseArgs(): '--sks-binary', default = paths['sksbin'], dest = 'sksbin', - help = 'The path to the SKS binary/executable to use to perform the dumps.') + help = ('The path to the SKS binary/executable to use ' + + 'to perform the dump.')) args.add_argument('-e', '--destdir', default = paths['destdir'], dest = 'destdir', - help = 'The directory where the dumps should be saved (a sub-directory with the date will be created).') + help = ('The directory where the dumps should be ' + + 'saved (a sub-directory with the date will be ' + + 'created).')) args.add_argument('-r', '--rsync', default = paths['rsync'], dest = 'rsync', - help = 'The remote (user@host:/path/) or local (/path/) path to use to sync the dumps to.') + help = ('The remote (user@host:/path/) or local '+ + '(/path/) path to use to sync the dumps to.')) args.add_argument('-t', '--throttle', default = float(sync['throttle']), dest = 'throttle', type = float, - help = 'The amount in KiB/s to throttle the rsync to. Use 0 for no throttling.') + help = ('The amount in KiB/s to throttle the rsync ' + + 'to. Use 0 for no throttling.')) args.add_argument('-D', '--no-dump', dest = 'nodump', @@ -300,7 +337,8 @@ def parseArgs(): dest = 'nocompress', action = 'store_true', default = ('nocompress' in runtime), - help = 'Don\'t compress the DB dumps (default is to compress)') + help = ('Don\'t compress the DB dumps (default is to ' + + 'compress)')) args.add_argument('-S', '--no-sync', dest = 'nosync', @@ -315,7 +353,8 @@ def main(): if getpass.getuser() not in ('root', args['user']): exit('ERROR: You must be root or {0}!'.format(args['user'])) with open(args['logfile'], 'a') as f: - f.write('===== {0} STARTING =====\n'.format(str(datetime.datetime.utcnow()))) + f.write('===== {0} STARTING =====\n'.format( + str(datetime.datetime.utcnow()))) if not args['nodump']: dumpDB(args) if not args['nocompress']: @@ -323,7 +362,8 @@ def main(): if not args['nosync']: syncDB(args) with open(args['logfile'], 'a') as f: - f.write('===== {0} DONE =====\n'.format(str(datetime.datetime.utcnow()))) + f.write('===== {0} DONE =====\n'.format( + str(datetime.datetime.utcnow()))) if __name__ == '__main__':