diff --git a/arch/repoclone.py b/arch/repoclone.py deleted file mode 100755 index 4c89021..0000000 --- a/arch/repoclone.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import configparser -import copy -import datetime -import os -import pprint -import subprocess -import sys - -# TODO: convert .ini to treat [section]s as repositories, with a [DEFAULT] -# section for URL etc. - -cfgfile = os.path.join(os.environ['HOME'], - '.config', - 'optools', - 'repoclone', - 'arch.ini') - -# Rsync options -opts = [ - '--recursive', # recurse into directories - '--times', # preserve modification times - '--links', # copy symlinks as symlinks - '--hard-links', # preserve hard links - '--quiet', # suppress non-error messages - '--delete-after', # receiver deletes after transfer, not during - '--delay-updates', # put all updated files into place at end - '--copy-links', # transform symlink into referent file/dir - '--safe-links', # ignore symlinks that point outside the tree - #'--max-delete', # don't delete more than NUM files - '--delete-excluded', # also delete excluded files from dest dirs - '--exclude=.*' # exclude files matching PATTERN - ] - -def sync(args): - # TODO: this should be a class, probably, instead as there's a lot of shared data across what should be multiple - # functions. - with open(os.devnull, 'w') as devnull: - mntchk = subprocess.run(['findmnt', args['mount']], stdout = devnull, stderr = devnull) - if mntchk.returncode != 0: - exit('!! BAILING OUT; {0} isn\'t mounted !!'.format(args['mount'])) - if args['bwlimit'] >= 1: - opts.insert(10, '--bwlimit=' + str(args['bwlimit'])) # limit socket I/O bandwidth - for k in ('destination', 'logfile', 'lockfile'): - os.makedirs(os.path.dirname(args[k]), exist_ok = True) - paths = os.environ['PATH'].split(':') - rsync = '/usr/bin/rsync' # set the default - for p in paths: - testpath = os.path.join(p, 'rsync') - if os.path.isfile(testpath): - rsync = testpath # in case rsync isn't in /usr/bin/rsync - break - cmd = [rsync] # the path to the binary - cmd.extend(opts) # the arguments - # TODO: implement repos here? - # end TODO - # The https://git.server-speed.net/users/flo/bin/tree/syncrepo.sh script uses http(s). to check for lastupdate. - # I don't, because not all mirrors *have* http(s). - check_cmd = copy.deepcopy(cmd) - check_cmd.append(os.path.join(args['mirror'], 'lastupdate')) - check_cmd.append(os.path.join(args['destination'], 'lastupdate')) - update_cmd = copy.deepcopy(cmd) - update_cmd.append(os.path.join(args['mirror'], 'lastsync')) - update_cmd.append(os.path.join(args['destination'], 'lastsync')) - cmd.append(os.path.join(args['mirror'], '.')) # the path on the remote mirror (full sync) - cmd.append(os.path.join(args['destination'], '.')) # the local destination (full sync) - if os.path.isfile(args['lockfile']): - with open(args['lockfile'], 'r') as f: - existingpid = f.read().strip() - if os.isatty(sys.stdin.fileno()): - # Running from shell - exit('!! A repo synchronization seems to already be running (PID: {0}). Quitting. !!'.format(existingpid)) - else: - exit() # we're running in cron, shut the hell up. - else: - with open(args['lockfile'], 'w') as f: - f.write(str(os.getpid())) - # determine if we need to do a full sync. - # TODO: clean this up. there's a lot of code duplication here, and it should really be a function. - with open(os.path.join(args['destination'], 'lastupdate'), 'r') as f: - oldupdate = datetime.datetime.utcfromtimestamp(int(f.read().strip())) - with open(os.devnull, 'wb') as devnull: - # TODO: when i clean this up, change this to do error detection - c = subprocess.run(check_cmd, stdout = devnull, stderr = devnull) - c2 = subprocess.run(update_cmd, stdout = devnull, stderr = devnull) - with open(os.path.join(args['destination'], 'lastupdate'), 'r') as f: - newupdate = datetime.datetime.utcfromtimestamp(int(f.read().strip())) - if newupdate > oldupdate: - with open(args['logfile'], 'a') as log: - c = subprocess.run(cmd, stdout = log, stderr = subprocess.PIPE) - now = int(datetime.datetime.timestamp(datetime.datetime.utcnow())) - with open(os.path.join(args['destination'], 'lastsync'), 'w') as f: - f.write(str(now) + '\n') - else: - # No-op. Stderr should be empty. - c = subprocess.run(['echo'], stdout = subprocess.PIPE, stderr = subprocess.PIPE) - now = int(datetime.datetime.timestamp(datetime.datetime.utcnow())) - with open(args['lastcheck'], 'w') as f: - f.write(str(now) + '\n') - os.remove(args['lockfile']) - # Only report errors at the end of the run if we aren't running in cron. Otherwise, log them. - errors = c.stderr.decode('utf-8').splitlines() - if os.isatty(sys.stdin.fileno()) and errors: - print('We encountered some errors:') - for e in errors: - if e.startswith('symlink has no referent: '): - print('Broken upstream symlink: {0}'.format(e.split()[1].replace('"', ''))) - else: - print(e) - elif errors: - with open(args['logfile'], 'a') as f: - for e in errors: - f.write('{0}\n'.format(e)) - return() - -def getDefaults(): - # Hardcoded defaults - dflt = {'mirror': 'rsync://mirror.square-r00t.net/arch/', - 'repos': 'core,extra,community,multilib,iso/latest', - 'destination': '/srv/repos/arch', - 'lastcheck': '/srv/http/arch.lastcheck', - 'mount': '/', - 'bwlimit': 0, - 'lockfile': '/var/run/repo-sync_arch.lck', - 'logfile': '/var/log/repo/arch.log'} - realcfg = configparser.ConfigParser(defaults = dflt) - if not os.path.isfile(cfgfile): - with open(cfgfile, 'w') as f: - realcfg.write(f) - realcfg.read(cfgfile) - return(realcfg) - -def parseArgs(): - cfg = getDefaults() - liveopts = cfg['DEFAULT'] - args = argparse.ArgumentParser(description = 'Synchronization for a remote Arch repository to a local one.', - epilog = ('This program will write a default configuration file to {0} ' + - 'if one is not found.').format(cfgfile)) - args.add_argument('-m', - '--mirror', - dest = 'mirror', - default = liveopts['mirror'], - help = ('The upstream mirror to sync from, must be an rsync URI '+ - '(Default: {0}').format(liveopts['mirror'])) -# TODO: can we do this? -# We can; we need to .format() a repo in, probably, on the src and dest. -# Problem is the last updated/last synced files. -# args.add_argument('-r', -# '--repos', -# dest = 'repos', -# default = liveopts['repos'], -# help = ('The repositories to sync; must be a comma-separated list. ' + -# '(Currently not used.) Default: {0}').format(','.join(liveopts['repos']))) - args.add_argument('-d', - '--destination', - dest = 'destination', - default = liveopts['destination'], - help = 'The destination directory to sync to. Default: {0}'.format(liveopts['destination'])) - args.add_argument('-c', '--last-check', - dest = 'lastcheck', - default = liveopts['lastcheck'], - help = ('The file to update with a timestamp on every run. Per spec, this must be outside the ' - 'repository webroot')) - args.add_argument('-b', - '--bwlimit', - dest = 'bwlimit', - default = liveopts['bwlimit'], - type = int, - help = ('The amount, in Kilobytes per second, to throttle the sync to. Default is to not ' - 'throttle (0).')) - args.add_argument('-l', - '--log', - dest = 'logfile', - default = liveopts['logfile'], - help = 'The path to the logfile. Default: {0}'.format(liveopts['logfile'])) - args.add_argument('-L', - '--lock', - dest = 'lockfile', - default = liveopts['lockfile'], - help = 'The path to the lockfile. Default: {0}'.format(liveopts['lockfile'])) - args.add_argument('-M', - '--mount', - dest = 'mount', - default = liveopts['mount'], - help = 'The mountpoint for your --destination. The script will exit if this point is not mounted. ' + - 'If you don\'t need mount checking, just use /. Default: {0}'.format(liveopts['mount'])) - return(args) - -def main(): - args = vars(parseArgs().parse_args()) - sync(args) - return() - -if __name__ == '__main__': - main() diff --git a/centos/repoclone/centos.dflts.ini b/centos/repoclone/centos.dflts.ini deleted file mode 100644 index be66fc0..0000000 --- a/centos/repoclone/centos.dflts.ini +++ /dev/null @@ -1,146 +0,0 @@ -# This is an example ~/.config/optools/repoclone/centos.ini. -# You may need to change some options, but they are all commented so you know -# what to replace. -############################################################################### -# SPECIAL VALUES # -# You may recognize some values as used in yum's repo files -# (i.e. /etc/yum.repos.d/*.repo). THEY ARE NOT THE SAME. You CANNOT and SHOULD -# NOT simply copy-and-paste them in here, as they are constructed/used -# differently. -# That said, the following are special values/variables that are generated -# automatically (*case-sensitive*): -# -# {name}: The name of the repo (i.e. what appears in [brackets]). -# It is best that you leave this set in [DEFAULT] the way it -# is. -# -# {cur_arch}: The current hardware architecture of the host running the -# script e.g. x86_64 or i686. -# -# {rel_ver}: The release version. This will substitute for each version -# in the repository's "releases" directive. For example: -# -# destination = /srv/repos/centos/{rel_ver}/os/x86_64 -# releases = 6,7 -# baseuri = mirrors.centos.org/{rel_ver}/os/x86_64 -# -# would clone from the following to the following: -# -# mirrors.centos.org/6/os/x86_64 => -# /srv/repos/centos/6/os/x86_64 -# -# mirrors.centos.org/7/os/x86_64 => -# /srv/repos/centos/7/os/x86_64 -# -# {arch}: Similar to {rel_ver} in that it iterates, but for each item -# in the "arches" directive. -# -# {cur_ver}: *IF* (and *only* if) you are running this script on CentOS -# itself, this will be a "meta-variable" containing the full -# version, major version, minor version, and revision. -# i.e. For CentOS 7.4.1708, {cur_ver} or {cur_ver.full} will -# both replace as "7.4.1708". {cur_ver.maj} will replace as -# "7", {cur.min} will replace as "4", and {cur.rev} will -# replace as "1708". If you use this mechanism and are NOT -# running the script on CentOS, an error will occur. -# -# You can also refer to directives themselves -- see below for an example of -# this. -############################################################################### -# The default section provides... well, defaults. All of these values can be -# overridden in each repository defined. If any of these are not specified, -# either in DEFAULT or in the repo section itself, then an error will occur. -[DEFAULT] - -# The name of the repository. By default, this (repo_name) is the name of the -# section. -# {name} will ALWAYS be the section name and cannot be changed. -repo_name = {name} - -# Whether we should sync this repository or not. -# To enable a repository, set this to one of: 1, yes, on, true -# To disable a repository, set this to one of: 0, no, off, false -enabled = 0 - -# The list of architectures to clone, separated by commas (if you have more -# than one). This is iterated over. -arches = i686,{arch} - -# The full path to the "base" of the repository where we'll be rsyncing from. -# As shown, if an "$" is in front of curly brackets, you can use another -# directive in the same section. To specify a directive from another section, -# you would use ${section:directive} (e.g. ${base:repo_name}) -# Note that this has the potential to generate multiple iterations. -# isomirrors_sort.py should offer package repository mirrors as well, so that -# may be handy to find a fast mirror. -# Note that the default probably will not work for you since you need to be -# whitelisted to use it. -baseuri = mirror.centos.org/centos/{rel_ver}/${repo_name}/{arch} - -# Where the clone should go. If you are using iterables, make sure you use them -# here, too, otherwise you'll most likely overwrite parts and end up with a -# totally broken repository! -# The parent directories will be created if necessary (assuming we have proper -# permissions). -destination = ${mount}/centos/{rel_ver}/${repo_name}/{arch} - -# Perform a check before we start to make sure this mountpoint has a device -# mounted at it. If you don't store your repository mirrors at a different -# mountpoint, just set this to "/" (without quotes). -mount = /mnt/repos - -# If set, throttle the transfer speeds down to this number of Kilobytes per -# second (KB/s, *not* kbps!). -# If it's set to 0, don't perform any throttling. -bwlimit = 0 - -# The lockfile for the repository. If this file is present, the clone will -# abort to avoid rsync/file differentiation conflicts. The parent directories -# will be created if necessary (assuming we have proper permissions). -lockfile = /var/run/repo-sync_{name}.lck - -# The logfile. This setting is actually for all repositories; changing it below -# will have no effect as all entries use the same log file. The parent -# directories will be created if necessary (assuming we have proper -# permissions). -logfile = /var/log/repo/centos.log - -# The releases to clone for. Note that we follow symlinks, so you should *NOT* -# include e.g. both 7 and 7.4.1708. This is also an iterable in the form of a -# comma-separated list (if you have more than one). -releases = 6,{cur_ver.maj} - -# A comma-separated list of paths/patterns on the rsync server to exclude (if -# you have more than one). Leave empty for no excludes (this should be fine if -# your "baseuri"s are explicit enough). -excludes = - -# This would be equivalent to cloning the [base] repository found in -# /etc/yum.repos.d/CentOS-Base.repo -[base] -repo_name = os -enabled = 1 - -# Likewise with [base], but with [updates] instead, etc. -[updates] -enabled = 1 - -[extras] -enabled = 1 - -[centosplus] -enabled = 1 - -# /etc/yum.repos.d/epel.repo (assuming you installed the epel-release package) -# Just like CentOS mirrors, you probably need to change this since they run a -# whitelist. -[epel] -enabled = 1 -baseuri = dl.fedoraproject.org::fedora-{name}0/{rel_ver}/{arch} -destination = ${mount}/centos/{name}/{rel_ver}/{arch} - -# It even works with non-RedHat-supplied repositories, too! As long as they -# offer rsync access. I *highly* recommend you check IUS out: https://ius.io/ -[ius] -enabled = 0 -baseuri = dl.iuscommunity.org/{name}/stable/CentOS/{rel_ver}/{arch} diff --git a/centos/repoclone/repoclone.py b/centos/repoclone/repoclone.py deleted file mode 100755 index c1fb6d1..0000000 --- a/centos/repoclone/repoclone.py +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/env python3 - -import configparser -import copy -import datetime -import importlib.util -import os -import platform -import re -import socket -import subprocess -import sys - -cfgfile = os.path.join(os.environ['HOME'], - '.config', - 'optools', - 'repoclone', - 'centos.ini') - -# Set up the logger. -_selfpath = os.path.abspath(os.path.realpath(os.path.expanduser(__file__))) -_logmodpath = os.path.abspath(os.path.join(_selfpath, - '..', '..', '..', - 'lib', - 'python', - 'logger.py')) -log_spec = importlib.util.spec_from_file_location('logger', _logmodpath) -logger = importlib.util.module_from_spec(log_spec) -log_spec.loader.exec_module(logger) - -_loglevel = 'warning' -#_loglevel = 'debug' - -class cur_ver(object): - def __init__(self): - # TODO: .dist() is deprecated, as is linux_distribution. - # switch to distro? https://pypi.org/project/distro - _distname = platform.dist()[0] - if not re.search('^CentOS( Linux)?$', _distname, re.IGNORECASE): - raise ValueError(('You have specified "{cur_ver}" in your ' + - 'config, but you are not running this script ' + - 'on CentOS!')) - _ver = platform.dist()[1].split('.') - self.full = '.'.join(_ver) - self.maj = int(_ver[0]) - self.min = int(_ver[1]) - self.rev = _ver[2] - - def __str__(self): - return(self.full) - -# Rsync options -opts = [ - '--recursive', # recurse into directories - '--times', # preserve modification times - '--links', # copy symlinks as symlinks - '--hard-links', # preserve hard links - '--quiet', # suppress non-error messages - '--delete-after', # receiver deletes after transfer, not during - '--delay-updates', # put all updated files into place at end - '--copy-links', # transform symlink into referent file/dir - '--safe-links', # ignore symlinks that point outside the tree - #'--max-delete', # don't delete more than NUM files - '--delete-excluded', # also delete excluded files from dest dirs - ] - -dflts = {'DEFAULT': {'repo_name': '{name}', - 'enabled': False, - 'arches': ['i686', 'x86_64'], - 'baseuri': ('mirror.centos.org/centos/{rel_ver}/' + - '${repo_name}/{arch}'), - 'destination': ('${mount}/centos/{rel_ver}/' + - '${repo_name}/{arch}'), - 'mount': '/mnt/repos', - 'bwlimit': 0, - 'lockfile': '/var/run/repo-sync_{name}.lck', - 'logfile': '/var/log/repo/centos.log', - 'releases': [6, 7], - 'excludes': None}, - 'base': {'repo_name': 'os', - 'enabled': True}, - 'updates': {'enabled': True}, - 'extras': {'enabled': True}, - 'centosplus': {'enabled': True}, - 'epel': {'enabled': True, - 'baseuri': ('dl.fedoraproject.org::fedora-{name}0/' + - '{rel_ver}/{arch}'), - 'destination': '${mount}/centos/{name}/{rel_ver}/{arch}'}, - 'ius': {'enabled': False, - 'baseuri': ('dl.iuscommunity.org/{name}/stable/CentOS/' + - '{rel_ver}/{arch}')}} - -class MirrorMgr(object): - def __init__(self): - self.cfg = configparser.ConfigParser( - interpolation = configparser.ExtendedInterpolation(), - defaults = dflts['DEFAULT'], - allow_no_value = True) - self.strvars = {'cur_ver': None, - 'name': None, - 'arches': [], - 'releases': [], - 'cur_arch': platform.machine(), - 'rel_ver': None, - 'arch': None} - if not os.path.isfile(cfgfile): - self.gen_cfg() - self.get_cfg() - self.chk_cur_ver() - self.parse_cfg() - self.log = logger.log(logfile = os.path.abspath(os.path.expanduser( - self.cfg['DEFAULT']['logfile'])), - logname = 'optools.repoclone.centos', - loglvl = _loglevel) - - def get_cfg(self): - with open(cfgfile, 'r') as f: - self.cfg_in = f.read() - return() - - def chk_cur_ver(self): - for line in self.cfg_in.splitlines(): - _line = line - # Strip out inline comments -- this is disabled by default(?). - #_line = re.sub('\s*(#|;).*$', '', line) - # Skip empty lines/comments. - if re.search('^\s*((#|;).*)?$', line): - continue - # Check to see if cur_ver is referenced. - if re.search('^.*{cur_ver}.*$', _line): - self.strvars['cur_ver'] = cur_ver() - break - return() - - def gen_cfg(self): - cfg = configparser.ConfigParser( - interpolation = configparser.ExtendedInterpolation(), - defaults = dflts['DEFAULT'], - allow_no_value = True) - for i in dflts.keys(): - if i != 'DEFAULT': - cfg[i] = copy.deepcopy(dflts[i]) - with open(cfgfile, 'w') as f: - cfg.write(f) - # And add the comment about how it's a stripped down default conf. - with open(cfgfile, 'r+') as f: - cfgdata = f.read() - f.seek(0, 0) - cmnt = ('# This is an autogenerated configuration file for ' + - 'r00t^s\'s OpTools CentOS\n# mirror script.\n# You ' + - 'should reference the fully commented version ' + - 'distributed with the script,\n# "centos.dflts.ini".\n\n') - f.write(cmnt + cfgdata) - print(('A configuration file has been automatically generated for ' + - 'you at {0}. You should review and customize it, because it ' + - 'most likely will not work out of the box.').format(cfgfile)) - exit('Exiting to give you the chance to customize it...') - return() - - def parse_cfg(self): - self.cfg.read_string(self.cfg_in) - return() - - def sync(self): - for repo in self.cfg.sections(): - # Skip disabled repos. - if not self.cfg.getboolean(repo, 'enabled'): - continue - self.repo = copy.deepcopy(dict(self.cfg[repo])) - self.strvars['name'] = repo - # This should be safe since the only thing that makes sense here is - # {cur_arch}, which we populate in __init__(). - self.strvars['arches'] = [i.strip() for i in \ - self.repo['arches'].format( - **self.strvars).split(',')] - self.strvars['releases'] = [i.strip() for i in \ - self.repo['releases'].format( - **self.strvars).split(',')] - for arch in self.strvars['arches']: - for rel_ver in self.strvars['releases']: - self._clear_tpl(repo, arch, rel_ver) - self._repo_chk(repo) - self._repo_sync(repo) - return() - - def _clear_tpl(self, repo, arch, rel_ver): - self.repo = copy.deepcopy(dict(self.cfg[repo])) - self.strvars['name'] = repo - # This should be safe since the only thing that makes sense here is - # {cur_arch}, which we populate in __init__(). - self.strvars['arches'] = [i.strip() for i in \ - self.repo['arches'].format( - **self.strvars).split(',')] - self.strvars['releases'] = [i.strip() for i in \ - self.repo['releases'].format( - **self.strvars).split(',')] - self.strvars['arch'] = arch - self.strvars['rel_ver'] = rel_ver - self.strvars['name'] = repo - self._repo_chk(repo) - return() - - def _repo_sync(self, repo): - # Reset the Rsync options - self.opts = opts - self.repo['bwlimit'] = float(self.repo['bwlimit']) - if self.repo['bwlimit'] > 0.0: - # limit socket I/O bandwidth - self.opts.append('--bwlimit=' + str(self.repo['bwlimit'])) - paths = os.environ['PATH'].split(':') - cmd = ['rsync'] # Set up a cmd list for subprocess - cmd.extend(opts) # The arguments for rsync - # The path on the remote mirror - _path = os.path.join('rsync://{0}'.format(self.repo['baseuri']), '.') - cmd.append(_path) - # The local destination - cmd.append(os.path.join(self.repo['destination'], '.')) - if os.path.isfile(self.repo['lockfile']): - with open(self.repo['lockfile'], 'r') as f: - existingpid = f.read().strip() - if os.isatty(sys.stdin.fileno()): - # Running from shell - exit(('!! A repo synchronization seems to already be ' + - 'running (PID: {0}). Quitting. !!').format(existingpid)) - else: - exit() # We're running in cron, shut the hell up. - else: - with open(self.repo['lockfile'], 'w') as f: - f.write(str(os.getpid())) - with open(self.repo['logfile'], 'a') as log: - c = subprocess.run(cmd, stdout = log, stderr = subprocess.PIPE) - now = int(datetime.datetime.utcnow().timestamp()) - with open(os.path.join(self.repo['destination'], - 'lastsync'), 'w') as f: - f.write(str(now) + '\n') - os.remove(self.repo['lockfile']) - # Only report errors at the end of the run if we aren't running in - # cron. Otherwise, log them. - errors = c.stderr.decode('utf-8').splitlines() - # CentOS 7 main doesn't have an i686. - if self.strvars['rel_ver'] == 7: - for e in errors[:]: - rgx = re.compile(('^rsync: change_dir.*/i[36]86(/|").*' + - 'failed:\s*No\s+such\s+file\s+or\s+' + - 'directory.*$')) - if rgx.search(e): - errors.remove(e) - for e in errors[:]: - if e.startswith(('rsync error: some files/attrs were not ' + - 'transferred (see previous errors)')): - errors.remove(e) - if os.isatty(sys.stdin.fileno()) and errors: - print('[{0}] We encountered some errors:'.format(repo)) - for e in errors: - if e.startswith('symlink has no referent: '): - print(('Broken upstream symlink: ' + - '{0}').format(e.split()[1].replace('"', ''))) - else: - print(e) - else: - for e in errors: - log.write('{0}\n'.format(e)) - return() - - def _repo_chk(self, repo): - def chkmnt(): - self.repo['mount'] = os.path.abspath( - os.path.expanduser( - self.repo['mount'].format( - **self.strvars))) - with open(os.devnull, 'w') as devnull: - mntchk = subprocess.run(['findmnt', - self.repo['mount']], - stdout = devnull, - stderr = devnull) - if mntchk.returncode != 0: - raise RuntimeError(('!! BAILING OUT; {0} isn\'t ' + - 'mounted !!').format(self.repo['mount'])) - return() - def chkrsync(): - _port = 873 - _open = False - self.repo['baseuri'] = re.sub('^\s*rsync://', - '', - self.repo['baseuri'].format( - **self.strvars), - re.IGNORECASE) - _raw_srv = self.repo['baseuri'].split('/')[0] - _split_srv = re.sub('::.*$', '', _raw_srv).split(':') - if len(_split_srv) >= 2: - _port = _split_srv[1] - for proto in (socket.AF_INET, socket.AF_INET6): - s = socket.socket(proto, socket.SOCK_STREAM) - chk = s.connect_ex((_split_srv[0], _port)) - if chk == 0: - _open = True - break - if os.isatty(sys.stdin.fileno()): - if not _open: - raise RuntimeError(('Rsync on host {0}:{1} is not ' + - 'accessible!').format(_split_srv[0], - _port)) - else: - exit() - return() - def chkdest(): - _dest = os.path.abspath( - os.path.expanduser( - self.cfg[repo]['destination'].format( - **self.strvars))) - self.repo['destination'] = _dest - os.makedirs(self.repo['destination'], exist_ok = True) - return() - def chkdest_files(): - for f in ('logfile', 'lockfile'): - _dest = os.path.abspath( - os.path.expanduser( - self.repo[f].format(**self.strvars))) - self.repo[f] = _dest - os.makedirs(os.path.dirname(self.repo[f]), exist_ok = True) - return() - def chkmisc(): - # Odds and ends. - pass - return() - # The Business-End(TM) - chkmnt() - chkrsync() - chkdest() - chkdest_files() - chkmisc() - return() - -def main(): - m = MirrorMgr() - m.sync() - -if __name__ == '__main__': - main() diff --git a/centos/repoclone/test.py b/centos/repoclone/test.py deleted file mode 100755 index c5e9c1b..0000000 --- a/centos/repoclone/test.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 - -import os - -selfpath = os.path.abspath(os.path.expanduser(__file__)) -print(selfpath) - -logmodpath = os.path.abspath(os.path.join(selfpath, '..', '..', '..', 'lib', 'python')) -print(logmodpath) diff --git a/libvirt/README b/libvirt/README new file mode 100644 index 0000000..4e181d3 --- /dev/null +++ b/libvirt/README @@ -0,0 +1 @@ +These projects/scripts have been moved to https://git.square-r00t.net/LibvirtTools/. diff --git a/libvirt/better_virsh.py b/libvirt/better_virsh.py deleted file mode 100755 index 1a8e488..0000000 --- a/libvirt/better_virsh.py +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -# import os -# import getpass -import re -## -import libvirt -# from lxml import etree - -# NOTE: docs URLS are super long. Extrapolate using following: -# docsurl = 'https://libvirt.org/docs/libvirt-appdev-guide-python/en-US/html' - -# TODO: flesh this out. only supports guests atm -# TODO: use openAuth? -# {docsurl}/libvirt_application_development_guide_using_python-Connections.html#idp13928160 - -# I would like to take the moment to point out that I did in three hours with exactly NO prior knowledge of the libvirt -# API what Red Hat couldn't do in four YEARS. https://bugzilla.redhat.com/show_bug.cgi?id=1244093 - - -def libvirt_callback(userdata, err): - # fucking worst design decision. - # https://stackoverflow.com/a/45543887/733214 - pass - - -# fucking worst design decision. -# https://stackoverflow.com/a/45543887/733214 -libvirt.registerErrorHandler(f = libvirt_callback, ctx = None) - - -class LV(object): - def __init__(self, uri, *args, **kwargs): - self.uri = uri - self.conn = None - self._args = args - self._kwargs = kwargs - - def _getTargets(self, target, regex = False, ttype = 'guest', - state = None, nocase = False, *args, **kwargs): - targets = [] - # TODO: ..._RUNNING as well? can add multiple flags - state_flags = {'guest': (libvirt.VIR_CONNECT_LIST_DOMAINS_ACTIVE, - libvirt.VIR_CONNECT_LIST_DOMAINS_INACTIVE), - 'net': (libvirt.VIR_CONNECT_LIST_NETWORKS_ACTIVE, - libvirt.VIR_CONNECT_LIST_NETWORKS_INACTIVE), - 'storage': (libvirt.VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE, - libvirt.VIR_CONNECT_LIST_STORAGE_POOLS_INACTIVE)} - re_flags = re.UNICODE # The default - if nocase: - re_flags += re.IGNORECASE - if not self.conn: - self.startConn() - search_funcs = {'guest': self.conn.listAllDomains, - 'net': self.conn.listAllNetworks, - 'storage': self.conn.listAllStoragePools} - if not regex: - ptrn = r'^{0}$'.format(target) - else: - ptrn = target - ptrn = re.compile(ptrn, re_flags) - if state == 'active': - flag = state_flags[ttype][0] - elif state == 'inactive': - flag = state_flags[ttype][1] - else: - flag = 0 - for t in search_funcs[ttype](flag): - if ptrn.search(t.name()): - targets.append(t) - targets.sort(key = lambda i: i.name()) - return(targets) - - def list(self, target, verbose = False, *args, **kwargs): - # {docsurl}/libvirt_application_development_guide_using_python-Guest_Domains-Information-Info.html - if not self.conn: - self.startConn() - targets = self._getTargets(target, **kwargs) - results = [] - # Each attr is a tuple; the name of the attribute and the key name the result should use (if defined) - attr_map = {'str': (('name', None), - ('OSType', 'os'), - ('UUIDString', 'uuid'), - ('hostname', None)), - 'bool': (('autostart', None), - ('hasCurrentSnapshot', 'current_snapshot'), - ('hasManagedSaveImage', 'managed_save_image'), - ('isActive', 'active'), - ('isPersistent', 'persistent'), - ('isUpdated', 'updated')), - 'int': (('ID', 'id'), - ('maxMemory', 'max_memory_KiB'), - ('maxVcpus', 'max_vCPUs'))} - for t in targets: - if not verbose: - results.append(t.name()) - else: - r = {} - for attrname, newkey in attr_map['str']: - keyname = (newkey if newkey else attrname) - try: - r[keyname] = str(getattr(t, attrname)()) - except libvirt.libvirtError: - r[keyname] = '(N/A)' - for attrname, newkey in attr_map['bool']: - keyname = (newkey if newkey else attrname) - try: - r[keyname] = bool(getattr(t, attrname)()) - except (libvirt.libvirtError, ValueError): - r[keyname] = None - for attrname, newkey in attr_map['int']: - keyname = (newkey if newkey else attrname) - try: - r[keyname] = int(getattr(t, attrname)()) - if r[keyname] == -1: - r[keyname] = None - except (libvirt.libvirtError, ValueError): - r[keyname] = None - results.append(r) - return(results) - - def restart(self, target, *args, **kwargs): - self.stop(target, state = 'active', **kwargs) - self.start(target, state = 'inactive', **kwargs) - return() - - def start(self, target, **kwargs): - if not self.conn: - self.startConn() - targets = self._getTargets(target, state = 'inactive', **kwargs) - for t in targets: - t.create() - return() - - def stop(self, target, force = False, *args, **kwargs): - if not self.conn: - self.startConn() - targets = self._getTargets(target, state = 'active', **kwargs) - for t in targets: - if not force: - t.shutdown() - else: - t.destroy() - return () - - def startConn(self): - self.conn = libvirt.open(self.uri) - return() - - def stopConn(self): - if self.conn: - self.conn.close() - self.conn = None - return() - - -def parseArgs(): - args = argparse.ArgumentParser(description = 'Some better handling of libvirt guests') - common_args = argparse.ArgumentParser(add_help = False) - common_args.add_argument('-u', '--uri', - dest = 'uri', - default = 'qemu:///system', - help = 'The URI for the libvirt to connect to. Default: qemu:///system') - common_args.add_argument('-r', '--regex', - action = 'store_true', - help = 'If specified, use a regex pattern for TARGET instead of exact match') - common_args.add_argument('-i', '--case-insensitive', - action = 'store_true', - dest = 'nocase', - help = 'If specified, match the target name/regex pattern case-insensitive') - common_args.add_argument('-T', '--target-type', - # choices = ['guest', 'net', 'storage'], - choices = ['guest'], - default = 'guest', - dest = 'ttype', - help = 'The type of TARGET') - common_args.add_argument('-t', '--target', - dest = 'target', - metavar = 'TARGET', - default = '.*', - help = ('The guest, network, etc. to manage. ' - 'If not specified, operate on all (respecting other filtering)')) - subparsers = args.add_subparsers(help = 'Operation to perform', - dest = 'oper', - metavar = 'OPERATION', - required = True) - start_args = subparsers.add_parser('start', help = 'Start the target(s)', parents = [common_args]) - restart_args = subparsers.add_parser('restart', help = 'Restart the target(s)', parents = [common_args]) - stop_args = subparsers.add_parser('stop', help = 'Stop ("destroy") the target(s)', parents = [common_args]) - stop_args.add_argument('-f', '--force', - dest = 'force', - action = 'store_true', - help = 'Hard poweroff instead of send a shutdown/ACPI powerdown signal') - list_args = subparsers.add_parser('list', help = 'List the target(s)', parents = [common_args]) - list_args.add_argument('-v', '--verbose', - dest = 'verbose', - action = 'store_true', - help = 'Display more output') - list_args.add_argument('-s', '--state', - dest = 'state', - choices = ['active', 'inactive'], - default = None, - help = 'Filter results by state. Default is all states') - return(args) - - -def main(): - args = parseArgs().parse_args() - varargs = vars(args) - lv = LV(**varargs) - f = getattr(lv, args.oper)(**varargs) - if args.oper == 'list': - if args.verbose: - import json - print(json.dumps(f, indent = 4, sort_keys = True)) - else: - print('\n'.join(f)) - return() - - -if __name__ == '__main__': - main()