From b4ee009465a548ca26bc6cfb4f1c736722f33dd4 Mon Sep 17 00:00:00 2001 From: brent s Date: Wed, 25 Oct 2017 02:50:45 -0400 Subject: [PATCH] borg backup script added. ready for testing --- storage/backups/borg/backup.py | 524 ++++++++++++++++++++++++ storage/backups/borg/backup2.py | 224 ---------- storage/backups/borg/sample.config.json | 35 ++ 3 files changed, 559 insertions(+), 224 deletions(-) create mode 100755 storage/backups/borg/backup.py delete mode 100755 storage/backups/borg/backup2.py create mode 100644 storage/backups/borg/sample.config.json diff --git a/storage/backups/borg/backup.py b/storage/backups/borg/backup.py new file mode 100755 index 0000000..9fa0dfc --- /dev/null +++ b/storage/backups/borg/backup.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 + +# TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html +# will they EVER release a public API? for now we'll just use subprocess since +# we import it for various prep stuff anyways. + +import argparse +import configparser +import datetime +import json +import logging +import logging.handlers +import os +import subprocess +import sys +try: + import pymysql # not stdlib; "python-pymysql" in Arch's AUR + has_mysql = True +except ImportError: + has_mysql = False +try: + from systemd import journal + has_systemd = True +except ImportError: + has_systemd = False + +### LOG LEVEL MAPPINGS ### +loglvls = {'critical': logging.CRITICAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG} + +### THE GUTS ### +class Backup(object): + def __init__(self, args): + self.args = args + ### DIRECTORIES ### + if self.args['oper'] == 'backup': + for d in (self.args['mysqldir'], self.args['stagedir']): + os.makedirs(d, exist_ok = True, mode = 0o700) + ### LOGGING ### + # Thanks to: + # https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/ + # https://stackoverflow.com/a/42604392 + # https://plumberjack.blogspot.com/2010/10/supporting-alternative-formatting.html + # and user K900_ on r/python for entertaining my very silly question. + self.logger = logging.getLogger(__name__) + self.logger.setLevel(loglvls[self.args['loglevel']]) + _logfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({asctime}; {filename}:{lineno})', + style = '{', + datefmt = '%Y-%m-%d %H:%M:%S') + _journalfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({filename}:{lineno})', + style = '{', + datefmt = '%Y-%m-%d %H:%M:%S') + handlers = [] + if self.args['disklog']: + os.makedirs(os.path.dirname(self.args['logfile']), + exist_ok = True, + mode = 0o700) + handlers.append(logging.handlers.RotatingFileHandler(self.args['logfile'], + encoding = 'utf8', + maxBytes = 100000, + backupCount = 1)) + if self.args['verbose']: + handlers.append(logging.StreamHandler()) + if has_systemd: + h = journal.JournalHandler() + h.setFormatter(_journalfmt) + h.setLevel(loglvls[self.args['loglevel']]) + self.logger.addHandler(h) + for h in handlers: + h.setFormatter(_logfmt) + h.setLevel(loglvls[self.args['loglevel']]) + self.logger.addHandler(h) + self.logger.debug('BEGIN INITIALIZATION') + ### CONFIG ### + if not os.path.isfile(self.args['cfgfile']): + self.logger.error('{0} does not exist'.format(self.args['cfgfile'])) + exit(1) + with open(self.args['cfgfile'], 'r') as f: + self.cfg = json.loads(f.read()) + ### END LOGGING ### + ### ARGS CLEANUP ### + self.logger.debug('VARS (before args cleanup): {0}'.format(vars())) + self.args['repo'] = [i.strip() for i in self.args['repo'].split(',')] + if 'all' in self.args['repo']: + self.args['repo'] = list(self.cfg['repos'].keys()) + for r in self.args['repo'][:]: + if r == 'all': + self.args['repo'].remove(r) + elif r not in self.cfg['repos'].keys(): + self.logger.warning('Repository {0} is not configured; skipping.'.format(r)) + self.args['repo'].remove(r) + self.logger.debug('VARS (after args cleanup): {0}'.format(vars())) + self.logger.debug('END INITIALIZATION') + ### END INIT ### + + def cmdExec(self, cmd, stdoutfh = None): + self.logger.debug('Running command: {0}'.format(' '.join(cmd))) + if self.args['dryryun']: + return() # no-op + if outfh: + _err = subprocess.run(cmd, stdout = stdoutfh, stderr = subprocess.PIPE).stderr.decode('utf-8').strip() + else: + with open(os.devnull, 'w') as DEVNULL: + _err = subprocess.run(cmd, stdout = DEVNULL, stderr = subprocess.PIPE).stderr.decode('utf-8').strip() + if _err != '': + self.logger.error('STDERR: {0} ({1})'.format(_out.stderr.decode('utf-8'), + ' '.join(cmd))) + return() + + def createRepo(self): + _env = os.environ.copy() + _env['BORG_RSH'] = self.cfg['config']['ctx'] + for r in self.args['repo']: + self.logger.info('[{0}]: BEGIN INITIALIZATION'.format(r)) + _cmd = ['borg', + 'init', + '-v', + '{0}@{1}:{2}'.format(self.cfg['config']['user'], + self.cfg['config']['host'], + r)] + _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] + # We don't use self.cmdExec() here either because + # again, custom env, etc. + self.logger.debug('VARS: {0}'.format(vars())) + if not self.args['dryrun']: + _out = subprocess.run(_cmd, + env = _env, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + _stdout = _out.stdout.decode('utf-8').strip() + _stderr = _out.stderr.decode('utf-8').strip() + self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout)) + if _stderr != '': + self.logger.error('[{0}]: STDERR: {1} ({2})'.format(r, + _stderr, + ' '.join(_cmd))) + del(_env['BORG_PASSPHRASE']) + self.logger.info('[{0}]: END INITIALIZATION'.format(r)) + return() + + def create(self): + _env = os.environ.copy() + _env['BORG_RSH'] = self.cfg['config']['ctx'] + self.logger.info('START: backup') + for r in self.args['repo']: + self.logger.info('[{0}]: BEGIN BACKUP'.format(r)) + if 'prep' in self.cfg['repos'][r].keys(): + for prep in self.cfg['repos'][r]['prep']: + self.logger.info('[{0}]: Running prepfunc {1}'.format(r, prep)) + eval('self.{0}'.format(prep)) # I KNOW, IT'S TERRIBLE. so sue me. + self.logger.info('[{0}]: Finished prepfunc {1}'.format(r, prep)) + _cmd = ['borg', + 'create', + '-v', '--stats', + '--compression', 'lzma,9'] + if 'excludes' in self.cfg['repos'][r].keys(): + for e in self.cfg['repos'][r]['excludes']: + _cmd.extend(['--exclude', e]) + _cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'], + self.cfg['config']['host'], + r, + self.args['archive'])) + for p in cfg['repos'][r]['paths']: + _cmd.append(p) + _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] + self.logger.debug('VARS: {0}'.format(vars())) + # We don't use self.cmdExec() here because we want to explicitly pass the env + # and format the log line differently. + self.logger.debug('[{0}]: Running command: {1}'.format(r, ' '.join(cmd))) + if not self.args['dryrun']: + _out = subprocess.run(_cmd, + env = _env, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + _stdout = _out.stdout.decode('utf-8').strip() + _stderr = _out.stderr.decode('utf-8').strip() + self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout)) + if _stderr != '': + self.logger.error('[{0}]: STDERR: {1} ({2})'.format(r, + _stderr, + ' '.join(_cmd))) + del(_env['BORG_PASSPHRASE']) + self.logger.info('[{0}]: END BACKUP'.format(r)) + self.logger.info('END: backup') + return() + + def miscBak(self, pkgr): + self.logger.info('BEGIN: miscBak()') + _cmd = None + for p in os.environ['PATH'].split(':'): + d = os.path.expanduser(p) + if os.path.isfile(os.path.join(d, pkgtool)): + _pkgr = pkgtool + self.logger.debug('Package tool found at {0}'.format(_pkgr)) + else: + _pkgr = 'pacman' + self.logger.debug('Using {0} as package tool'.format(_pkgr)) + with open(os.path.join(self.args['stagedir'], 'pkg.lst'), 'w') as f: + _cmd = [_pkgr, + '-Qet', + '--color', + 'never'] + self.cmdExec(_cmd, stdoutfh = f) + self.logger.info('END: miscBak()') + return() + + def mysqlBak(self): + self.logger.info('BEGIN: mysqlBak()') + if not has_mysql: + self.logger.error('You need to install the PyMySQL module to back up MySQL databases. Skipping.') + return() + # These are mysqldump options shared by ALL databases + _mysqlopts = ['--routines', + '--add-drop-database', + '--add-drop-table', + '--allow-keywords', + '--complete-insert', + '--create-options', + '--extended-insert'] + _DBs = [] + _mycnf = os.path.expanduser(os.path.join('~', '.my.cnf')) + if not os.path.isfile(_mycnf): + exit('{0}: ERROR: Cannot get credentials for MySQL (cannot find ~/.my.cnf)!') + _mycfg = configparser.ConfigParser() + _mycfg._interpolation = configparser.ExtendedInterpolation() + _mycfg.read(_mycnf) + _sqlcfg = {s:dict(_mycfg.items(s)) for s in _mycfg.sections()} + if 'host' not in _sqlcfg.keys(): + _socketpath = '/var/run/mysqld/mysqld.sock' # correct for Arch, YMMV. + _mysql = pymysql.connect(unix_socket = _socketpath, + user = _sqlcfg['client']['user'], + passwd = _sqlcfg['client']['password']) + else: + _mysql = pymysql.connect(host = _sqlcfg['client']['host'], + user = _sqlcfg['client']['user'], + port = _sqlcfg['client']['port'], + passwd = _sqlcfg['client']['password']) + _cur = _mysql.cursor() + _cur.execute('SHOW DATABASES') + for row in _cur.fetchall(): + _DBs.append(row[0]) + self.logger.debug('Databases: {0}'.format(', '.join(_DBs))) + for db in _DBs: + _cmd = ['mysqldump', + '--result-file={0}.sql'.format(os.path.join(mysqldir, db))] + # These are database-specific options + if db in ('information_schema', 'performance_schema'): + _cmd.append('--skip-lock-tables') + elif db == 'mysql': + _cmd.append('--flush-privileges') + _cmd.extend(_mysqlopts) + _cmd.append(db) + self.cmdExec(_cmd) + self.logger.info('END: mysqlBak()') + return() + + def listRepos(self): + print('\n\033[1mCurrently configured repositories are:\033[0m\n') + print('\t{0}\n'.format(', '.join(self.cfg['repos'].keys()))) + if self.args['verbose']: + print('\033[1mDETAILS:\033[0m\n') + for r in self.args['repo']: + print('\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(r.upper()), end = '') + for p in self.cfg['repos'][r]['paths']: + print(p, end = ' ') + if 'prep' in self.cfg['repos'][r].keys(): + print('\n\t\t\033[1mPrep:\033[0m\t\t', end = '') + for p in self.cfg['repos'][r]['prep']: + print(p, end = ' ') + if 'excludes' in self.cfg['repos'][r].keys(): + print('\n\t\t\033[1mExclude(s):\033[0m\t', end = '') + for p in self.cfg['repos'][r]['excludes']: + print(p, end = ' ') + print('\n') + return() + + def printer(self): + _results = self.lister() + if not self.args['archive']: # It's a listing of archives + print('\033[1mREPO:\tSNAPSHOT:\t\tTIMESTAMP:\033[0m\n') + for r in _results.keys(): + print(r, end = '') + for line in _results[r]: + _snapshot = line.split() + print('\t\033[1m{0}\033[0m\t\t{1}'.format(_snapshot[0], ' '.join(_snapshot[1:]))) + print() + else: # It's a listing inside an archive + if self.args['verbose']: + fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', 'TIMESTAMP:', 'PATH:'] + for r in results.keys(): + print('\033[1m{0}\t{1}\033[0m'.format(fields[0], r)) + # https://docs.python.org/3/library/string.html#formatspec + print('{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(fields)) + for line in _results[r]: + _fline = line.split() + _perms = fline[0] + _ownership = '{0}:{1}'.format(fline[1], fline[2]) + _size = fline[3] + _time = ' '.join(fline[4:7]) + _path = ' '.join(fline[7:]) + print('{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(_perms, + _ownership, + _size, + _time, + _path)) + else: + print('\033[1mREPO:\tPATH:\033[0m\n') + for r in _results.keys(): + print(r, end = '') + for line in _results[r]: + _fline = line.split() + print('\t{0}'.format(' '.join(_fline[7:]))) + return() + + def lister(self): + output = {} + _env = os.environ.copy() + self.logger.info('START: lister') + _env['BORG_RSH'] = self.cfg['config']['ctx'] + for r in self.args['repo']: + if self.args['archive']: + _cmd = ['borg', + 'list', + '{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'], + self.cfg['config']['host'], + r, + self.args['archive'])] + else: + _cmd = ['borg', + 'list', + '{0}@{1}:{2}'.format(self.cfg['config']['user'], + self.cfg['config']['host'], + r)] + _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] + if not self.args['dryrun']: + + _out = subprocess.run(_cmd, + env = _env, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + _stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()] + _stderr = _out.stderr.decode('utf-8').strip() + output[r] = _stdout + self.logger.debug('[{0}]: (RESULT) {1}'.format(r, + '\n'.join(_stdout))) + if _stderr != '': + self.logger.error('[{0}]: STDERR: {1} ({2})'.format(r, + _stderr, + ' '.join(_cmd))) + del(_env['BORG_PASSPHRASE']) + if not self.args['archive']: + if self.args['numlimit'] > 0: + if self.args['old']: + output[r] = output[r][:self.args['numlimit']] + else: + output[r] = list(reversed(output[r]))[:self.args['numlimit']] + if self.args['invert']: + output[r] = reversed(output[r]) + self.logger.info('END: lister') + return(output) + +def printMoarHelp(): + _helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n' + + 'In order to efficiently display results, there are several options to handle it. ' + + 'Namely, these are:\n\n\t\t-s/--sort [direction]\n\t\t-l/--limit [number]\n\t\t-x/--invert\n\n' + + 'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t-l 5\n\n' + + 'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently ' + + 'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t-l 5 -x\n\n' + + 'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order (that is, ' + + 'sorted from newest to oldest), that\'d be: \n\n\t\t-o -l 7 -x\n') + print(_helpstr) + exit(0) + +def parseArgs(): + ### DEFAULTS ### + _date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M") + _logfile = '/var/log/borg/{0}'.format(_date) + _mysqldir = os.path.abspath(os.path.join(os.path.expanduser('~'), 'bak', 'mysql')) + _stagedir = os.path.abspath(os.path.join(os.path.expanduser('~'), '.bak', 'misc')) + _cfgfile = os.path.abspath(os.path.join(os.path.expanduser('~'), '.config', 'optools', 'backup.json')) + _defloglvl = 'info' + ###### + args = argparse.ArgumentParser(description = 'Backups manager', + epilog = 'TIP: this program has context-specific help. e.g. try "%(prog)s list --help"') + args.add_argument('-c', '--config', + dest = 'cfgfile', + default = _cfgfile, + help = ('The path to the config file. Default: \033[1m{0}\033[0m'.format(_cfgfile))) + args.add_argument('-Ll', '--loglevel', + dest = 'loglevel', + default = _defloglvl, + choices = list(loglvls.keys()), + help = ('The level of logging to perform. \033[1mWARNING:\033[0m \033[1mdebug\033[0m will log ' + + 'VERY sensitive information such as passwords! Default: \033[1m{0}\033[0m'.format(_defloglvl))) + args.add_argument('-Ld', '--log-to-disk', + dest = 'disklog', + action = 'store_true', + help = ('If specified, log to a specific file (-Lf/--logfile)' + + ' instead of the system logger.')) + args.add_argument('-Lf', '--logfile', + dest = 'logfile', + default = _logfile, + help = ('The path to the logfile, only used if -Ld/--log-to-disk ' + + 'is specified. Default: \033[1m{0}\033[0m (dynamic)').format(_logfile)) + args.add_argument('-v', '--verbose', + dest = 'verbose', + action = 'store_true', + help = ('If specified, log messages will be printed to STDERR ' + + 'in addition to the other configured log system(s), and verbosity for printing ' + + 'functions is increased. \033[1mWARNING:\033[0m This may display VERY sensitive information ' + + 'such as passwords!')) + ### ARGS FOR ALL OPERATIONS ### + commonargs = argparse.ArgumentParser(add_help = False) + commonargs.add_argument('-r', '--repo', + dest = 'repo', + default = 'all', + help = ('The repository to perform the operation for. ' + + 'The default is \033[1mall\033[0m, a special value that specifies all known ' + + 'repositories. Can also accept a comma-separated list.')) + remoteargs = argparse.ArgumentParser(add_help = False) + remoteargs.add_argument('-d', '--dry-run', + dest = 'dryrun', + action = 'store_true', + help = ('Act as if we are performing tasks, but none will actually be executed ' + + '(useful for testing logging)')) + ### OPERATIONS ### + subparsers = args.add_subparsers(help = 'Operation to perform', + dest = 'oper') + backupargs = subparsers.add_parser('backup', + help = 'Perform a backup.', + parents = [commonargs, remoteargs]) + listargs = subparsers.add_parser('list', + help = 'List available backups.', + parents = [commonargs, remoteargs]) + listrepoargs = subparsers.add_parser('listrepos', + help = 'List availabile/configured repositories.', + parents = [commonargs]) + initargs = subparsers.add_parser('init', + help = 'Initialise a repository.', + parents = [commonargs, remoteargs]) + ### OPERATION-SPECIFIC OPTIONS ### + # CREATE ("backup") # + backupargs.add_argument('-a', + '--archive', + default = _date, + dest = 'archive', + help = ('The name of the archive. Default: \033[1m{0}\033[0m (dynamic)').format(_date)) + backupargs.add_argument('-s', + '--stagedir', + default = _stagedir, + dest = 'stagedir', + help = ('The directory used for staging temporary files, ' + + 'if necessary. Default: \033[1m{0}\033[0m').format(_stagedir)) + backupargs.add_argument('-m', + '--mysqldir', + default = _mysqldir, + dest = 'mysqldir', + help = ('The path to where MySQL dumps should go. ' + + 'Default: \033[1m{0}\033[0m').format(_mysqldir)) + # DISPLAY/OUTPUT ("list") # + listargs.add_argument('-a', + '--archive', + dest = 'archive', + default = False, + help = 'If specified, will list the *contents* of the given archive name.') + listargs.add_argument('-l', + '--limit', + dest = 'numlimit', + type = int, + default = '5', + help = ('If specified, constrain the outout to this number of ' + + 'results each repo. Default is \033[1m5\033[0m, use 0 for unlimited. ' + + 'See \033[1m-H/--list-help\033[0m')) + listargs.add_argument('-s', + '--sort', + dest = 'sortby', + choices = ['newest', 'oldest'], + default = 'oldest', + help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. ' + + 'Default: \033[1moldest\033[0m')) + listargs.add_argument('-x', + '--invert', + dest = 'invert', + action = 'store_true', + help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.') + listargs.add_argument('-o', + '--old', + dest = 'old', + action = 'store_true', + help = ('Instead of grabbing the latest results, grab the earliest results. ' + + 'This differs from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.')) + listargs.add_argument('-H', + '--list-help', + dest = 'moarhelp', + action = 'store_true', + help = 'Print extended information about how to manage the output of listing and exit.') + return(args) + +def main(): + rawargs = parseArgs() + args = vars(rawargs.parse_args()) + args['cfgfile'] = os.path.abspath(os.path.expanduser(args['cfgfile'])) + if not args['oper']: + rawargs.print_help() + exit(0) + if 'moarhelp' in args.keys() and args['moarhelp']: + printMoarHelp() + # The "Do stuff" part + bak = Backup(args) + if args['oper'] == 'list': + bak.printer() + elif args['oper'] == 'listrepos': + bak.listRepos() + elif args['oper'] == 'backup': + bak.createBak() + elif args['oper'] == 'init': + bak.createRepo() + return() + +if __name__ == '__main__': + main() diff --git a/storage/backups/borg/backup2.py b/storage/backups/borg/backup2.py deleted file mode 100755 index 98b8f60..0000000 --- a/storage/backups/borg/backup2.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python3 - -# TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html -# will they EVER release a public API? for now we'll just use subprocess since -# we import it for various prep stuff anyways. - -import argparse -import configparser -import datetime -import json -import logging -import logging.handlers -import os -import subprocess -import sys -try: - import pymysql # not stdlib; "python-pymysql" in Arch's AUR - has_mysql = True -except ImportError: - has_mysql = False -try: - from systemd import journal - has_systemd = True -except ImportError: - has_systemd = False - -### LOG LEVEL MAPPINGS ### -loglvls = {'critical': logging.CRITICAL, - 'error': logging.ERROR, - 'warning': logging.WARNING, - 'info': logging.INFO, - 'debug': logging.DEBUG} - -### THE GUTS ### -class Backup(object): - def __init__(self, args): - self.args = args - ### LOGGING ### - # Thanks to: - # https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/ - # https://stackoverflow.com/a/42604392 - # https://plumberjack.blogspot.com/2010/10/supporting-alternative-formatting.html - # and user K900_ on r/python for entertaining my very silly question. - self.logger = logging.getLogger(__name__) - self.logger.setLevel(loglvls[self.args['loglevel']]) - _logfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({asctime}; {filename}:{lineno})', - style = '{', - datefmt = '%Y-%m-%d %H:%M:%S') - _journalfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({filename}:{lineno})', - style = '{', - datefmt = '%Y-%m-%d %H:%M:%S') - handlers = [] - if self.args['disklog']: - handlers.append(logging.handlers.RotatingFileHandler(self.args['logfile'], - encoding = 'utf8', - maxBytes = 100000, - backupCount = 1)) - if self.args['verbose']: - handlers.append(logging.StreamHandler()) - if has_systemd: - handlers.append(journal.JournalHandler()) - for h in handlers: - h.setFormatter(_logfmt) - h.setLevel(loglvls[self.args['loglevel']]) - self.logger.addHandler(h) - ### CONFIG ### - if not os.path.isfile(self.args['cfgfile']): - self.logger.error('{0} does not exist'.format(self.args['cfgfile'])) - exit(1) - - def create(self): - pass - - def miscBak(self, pkgr): - pass - - def mysqlBak(self): - pass - - def listRepos(self): - pass - - def printer(self): - pass - - def lister(self): - pass - -def printMoarHelp(): - helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n' + - 'In order to efficiently display results, there are several options to handle it. ' + - 'Namely, these are:\n\n\t\t-s/--sort [direction]\n\t\t-l/--limit [number]\n\t\t-x/--invert\n\n' + - 'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t-l 5\n\n' + - 'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently ' + - 'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t-l 5 -x\n\n' + - 'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order (that is, ' + - 'sorted from newest to oldest), that\'d be: \n\n\t\t-o -l 7 -x\n') - print(helpstr) - exit(0) - -def parseArgs(): - ### DEFAULTS ### - _date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M") - _logfile = '/var/log/borg/{0}'.format(_date) - _mysqldir = os.path.abspath(os.path.join(os.path.expanduser('~'), 'bak', 'mysql')) - _stagedir = '/root/bak/misc' - _cfgfile = os.path.abspath(os.path.join(os.path.expanduser('~'), '.config', 'optools', 'backup.json')) - _defloglvl = 'info' - ###### - args = argparse.ArgumentParser(description = 'Backups manager', - epilog = 'TIP: this program has context-specific help. e.g. try "%(prog)s list --help"') - args.add_argument('-c', '--config', - dest = 'cfgfile', - default = _cfgfile, - help = ('The path to the config file. Default: \033[1m{0}\033[0m'.format(_cfgfile))) - args.add_argument('-Ll', '--loglevel', - dest = 'loglevel', - default = _defloglvl, - choices = list(loglvls.keys()), - help = 'The level of logging to perform. Default: \033[1m{0}\033[0m'.format(_defloglvl)) - args.add_argument('-Ld', '--log-to-disk', - dest = 'disklog', - action = 'store_true', - help = ('If specified, log to a specific file (-Lf/--logfile)' + - ' instead of the system logger.')) - args.add_argument('-Lf', '--logfile', - dest = 'logfile', - default = _logfile, - help = ('The path to the logfile, only used if -Ld/--log-to-disk ' + - 'is specified. Default: \033[1m{0}\033[0m (dynamic)').format(_logfile)) - args.add_argument('-v', '--verbose', - dest = 'verbose', - action = 'store_true', - help = ('If specified, log messages will be printed to STDOUT/STDERR ' + - 'in addition to the other configured log systems.')) - ### ARGS FOR ALL OPERATIONS ### - commonargs = argparse.ArgumentParser(add_help = False) - commonargs.add_argument('-r', '--repo', - dest = 'repo', - default = 'all', - help = ('The repository to perform the operation for. ' + - 'The default is \033[1mall\033[0m, a special value that specifies all known ' + - 'repositories. Can also accept a comma-separated list.')) - ### OPERATIONS ### - subparsers = args.add_subparsers(help = 'Operation to perform', - dest = 'oper') - backupargs = subparsers.add_parser('backup', - help = 'Perform a backup.', - parents = [commonargs]) - listargs = subparsers.add_parser('list', - help = 'List available backups.', - parents = [commonargs]) - listrepoargs = subparsers.add_parser('listrepos', - help = 'List availabile/configured repositories.', - parents = [commonargs]) - initargs = subparsers.add_parser('init', - help = 'Initialise a repository.', - parents = [commonargs]) - ### OPERATION-SPECIFIC OPTIONS ### - # CREATE ("backup") # - backupargs.add_argument('-a', - '--archive', - default = _date, - dest = 'archive', - help = ('The name of the archive. Default: \033[1m{0}\033[0m (dynamic)').format(_date)) - backupargs.add_argument('-s', - '--stagedir', - default = _stagedir, - dest = 'stagedir', - help = ('The directory used for staging temporary files, ' + - 'if necessary. Default: \033[1m{0}\033[0m').format(_stagedir)) - # DISPLAY/OUTPUT ("list") # - listargs.add_argument('-a', - '--archive', - dest = 'archive', - default = False, - help = 'If specified, will list the *contents* of a certain archive.') - listargs.add_argument('-l', - '--limit', - dest = 'numlimit', - type = int, - default = '5', - help = ('If specified, constrain the outout to this number of ' + - 'results each repo. Default is \033[1m5\033[0m, use 0 for unlimited. ' + - 'See \033[1m-H/--list-help\033[0m')) - listargs.add_argument('-s', - '--sort', - dest = 'sortby', - choices = ['newest', 'oldest'], - default = 'oldest', - help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. ' + - 'Default: \033[1moldest\033[0m')) - listargs.add_argument('-x', - '--invert', - dest = 'invert', - action = 'store_true', - help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.') - listargs.add_argument('-o', - '--old', - dest = 'old', - action = 'store_true', - help = ('Instead of grabbing the latest results, grab the earliest results. ' + - 'This differs from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.')) - listargs.add_argument('-H', - '--list-help', - dest = 'moarhelp', - action = 'store_true', - help = 'Print extended information about how to manage the output of listing and exit.') - return(args) - -def main(): - rawargs = parseArgs() - args = vars(rawargs.parse_args()) - import pprint # DEBUG - pprint.pprint(args) # DEBUG - if not args['oper']: - rawargs.print_help() - exit(0) - if 'moarhelp' in args.keys() and args['moarhelp']: - printMoarHelp() - bak = Backup(args) - -if __name__ == '__main__': - main() diff --git a/storage/backups/borg/sample.config.json b/storage/backups/borg/sample.config.json new file mode 100644 index 0000000..7c31a33 --- /dev/null +++ b/storage/backups/borg/sample.config.json @@ -0,0 +1,35 @@ +# A normal config would be placed in the current user's come directory, +# typically under ~/.config/optools/backup.json + +{ + "repos": { + "web": { + "paths": ["/srv/http"], + "password": "PASSWORD1", # OBVIOUSLY, change all these. + "excludes": ["/srv/http/sks/dumps"] # you can specify paths to exclude. + }, + "etc": { + "paths": ["/etc"], + "password": "PASSWORD2" + }, + "home": { + "paths": ["/home"], + "password": "PASSWORD3" + }, + "misc": { + "paths": ["/usr/local/bin", "/var/lib/awstats", "/var/spool/anacron", "/var/spool/cron", "/usr/local/bin", "/root/.bak/misc"], + "prep": ["miscBak('apacman')"], # i'm... thinking of ways to make this more flexible, but for now it supports pacman/apacman + "password": "PASSWORD4" + }, + "root": { + "paths": ["/root"], + "password": "PASSWORD5", + "excludes": ["/root/.cache", "/root/.bak", "*.sql*"] + } + }, + "config": { + "user": "username", # the username on your backup box + "host": "backups.domain.tld", # where the borg server is + "ctx": "ssh -p 1234" # if you run a borg-specific SSHD on an alternate port + } +} \ No newline at end of file