From 4ef4a939e82d6ce1713f8d2427fee68df0694e48 Mon Sep 17 00:00:00 2001 From: brent s Date: Fri, 31 May 2019 12:28:07 -0400 Subject: [PATCH] think it's working now --- storage/backups/borg/plugins/yum_pkgs.py | 117 +++++++++++ .../backups/borg/tools/restore_yum_pkgs.py | 181 ++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 storage/backups/borg/plugins/yum_pkgs.py create mode 100755 storage/backups/borg/tools/restore_yum_pkgs.py diff --git a/storage/backups/borg/plugins/yum_pkgs.py b/storage/backups/borg/plugins/yum_pkgs.py new file mode 100644 index 0000000..1d843bc --- /dev/null +++ b/storage/backups/borg/plugins/yum_pkgs.py @@ -0,0 +1,117 @@ +import datetime +import os +import re +import sys +## +from lxml import etree +import yum + + +# Detect RH version. +ver_re =re.compile('^(centos.*|red\s?hat.*) ([0-9\.]+) .*$', re.IGNORECASE) +# distro module isn't stdlib, and platform.linux_distribution() (AND platform.distro()) are both deprecated in 3.7. +# So we get hacky. +with open('/etc/redhat-release', 'r') as f: + rawver = f.read() +distver = [int(i) for i in ver_re.sub('\g<2>', rawver.strip()).split('.')] +distname = re.sub('(Linux )?release', '', ver_re.sub('\g<1>', rawver.strip()), re.IGNORECASE).strip() +# Regex pattern to get the repo name. We compile it just to speed up the execution. +repo_re = re.compile('^@') +# Python version +pyver = sys.hexversion +py3 = 0x30000f0 # TODO: check the version incompats + + +class Backup(object): + def __init__(self, explicit_only = True, + include_deps = False, + output = '~/.cache/backup/installed_pkgs.xml'): + self.explicit_only = explicit_only + self.include_deps = include_deps + self.reasons = [] + if self.explicit_only: + self.reasons.append('user') + if self.include_deps: + self.reasons.append('dep') + self.output = os.path.abspath(os.path.expanduser(output)) + self.yb = yum.YumBase() + # Make it run silently. + self.yb.preconf.debuglevel = 0 + self.yb.preconf.errorlevel = 0 + self.pkg_meta = [] + # TODO: XSD? + self.pkgs = etree.Element('packages') + self.pkgs.attrib['distro'] = distname + self.pkgs.attrib['version'] = '.'.join([str(i) for i in distver]) + self.pkglist = b'' + self.getPkgList() + self.buildPkgInfo() + self.write() + + def getPkgList(self): + if not self.explicit_only: + self.pkg_meta = self.yb.rpmdb.returnPackages() + else: + for pkg in self.yb.rpmdb.returnPackages(): + reason = pkg.yumdb_info.get('reason') + if reason and reason.lower() in self.reasons: + self.pkg_meta.append(pkg) + return() + + def buildPkgInfo(self): + for pkg in self.pkg_meta: + reponame = repo_re.sub('', pkg.ui_from_repo) + repo = self.pkgs.xpath('repo[@name="{0}"]'.format(reponame)) + if repo: + repo = repo[0] + else: + repo = etree.Element('repo') + repo.attrib['name'] = reponame + try: + repoinfo = self.yb.repos.repos[reponame] + repo.attrib['urls'] = '>'.join(repoinfo.urls) # https://stackoverflow.com/a/13500078 + repo.attrib['desc'] = repoinfo.name + repo.attrib['enabled'] = ('true' if repoinfo in self.yb.repos.listEnabled() else 'false') + except KeyError: # Repo is missing + repo.attrib['desc'] = '(metadata missing)' + self.pkgs.append(repo) + pkgelem = etree.Element('package') + pkginfo = {'name': pkg.name, + 'desc': pkg.summary, + 'version': pkg.ver, + 'release': pkg.release, + 'arch': pkg.arch, + 'built': datetime.datetime.fromtimestamp(pkg.buildtime), + 'installed': datetime.datetime.fromtimestamp(pkg.installtime), + 'sizerpm': pkg.packagesize, + 'sizedisk': pkg.installedsize, + 'NEVRA': pkg.nevra} + for k, v in pkginfo.items(): + if pyver >= py3: + pkgelem.attrib[k] = str(v) + else: + if isinstance(v, (int, long, datetime.datetime)): + pkgelem.attrib[k] = str(v).encode('utf-8') + elif isinstance(v, str): + pkgelem.attrib[k] = v.decode('utf-8') + else: + pkgelem.attrib[k] = v.encode('utf-8') + repo.append(pkgelem) + self.pkglist = etree.tostring(self.pkgs, + pretty_print = True, + xml_declaration = True, + encoding = 'UTF-8') + return() + + def write(self): + outdir = os.path.dirname(self.output) + if pyver >= py3: + os.makedirs(outdir, exist_ok = True) + os.chmod(outdir, mode = 0o0700) + else: + if not os.path.isdir(outdir): + os.makedirs(outdir) + os.chmod(outdir, 0o0700) + with open(self.output, 'wb') as f: + f.write(self.pkglist) + return() diff --git a/storage/backups/borg/tools/restore_yum_pkgs.py b/storage/backups/borg/tools/restore_yum_pkgs.py new file mode 100755 index 0000000..3a97426 --- /dev/null +++ b/storage/backups/borg/tools/restore_yum_pkgs.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python + +import argparse +import os +import re +import subprocess +import sys +import warnings +## +# The yum API is *suuuper* cantankerous and kind of broken, even. +# Patches welcome, but for now we just use subprocess. +import yum +from lxml import etree + + +# Detect RH version. +ver_re =re.compile('^(centos.*|red\s?hat.*) ([0-9\.]+) .*$', re.IGNORECASE) +# distro module isn't stdlib, and platform.linux_distribution() (AND platform.distro()) are both deprecated in 3.7. +# So we get hacky. +with open('/etc/redhat-release', 'r') as f: + rawver = f.read() +distver = [int(i) for i in ver_re.sub('\g<2>', rawver.strip()).split('.')] +distname = re.sub('(Linux )?release', '', ver_re.sub('\g<1>', rawver.strip()), re.IGNORECASE).strip() +# Regex pattern to get the repo name. We compile it just to speed up the execution. +repo_re = re.compile('^@') +# Python version +pyver = sys.hexversion +py3 = 0x30000f0 # TODO: check the version incompats + +if pyver < py3: + import copy + + +class Reinstaller(object): + def __init__(self, pkglist_path, latest = True): + self.latest = latest + pkglist_file = os.path.abspath(os.path.expanduser(pkglist_path)) + with open(pkglist_file, 'rb') as f: + self.pkgs = etree.fromstring(f.read()) + if not self.latest: + # Make sure the versions match, otherwise Bad Things(TM) can occur. + if not all(((distname == self.pkgs.attrib['distro']), + ('.'.join([str(i) for i in distver]) == self.pkgs.attrib['version']))): + err = ('This package set was created on {0} {1}. ' + 'The current running OS is {2} {3} and you have set latest = False/None. ' + 'THIS IS A VERY BAD IDEA.').format(self.pkgs.attrib['distro'], + self.pkgs.attrib['version'], + distname, + '.'.join([str(i) for i in distver])) + raise RuntimeError(err) + # Make it run silently. + self.yb = yum.YumBase() + self.yb.preconf.quiet = 1 + self.yb.preconf.debuglevel = 0 + self.yb.preconf.errorlevel = 0 + self.yb.preconf.assumeyes = 1 + self.yb.preconf.rpmverbosity = 'error' + + def iterPkgs(self): + for repo in self.pkgs.findall('repo'): + # Base install packages ("anaconda") don't play nicely with this. They should be expected to + # already be installed anyways, and self.latest is irrelevant - downgrading these can cause + # *major* issues. + # And "installed" repo are packages installed manually from RPM. + if self.latest: + if repo.attrib['name'].lower() in ('anaconda', 'installed'): + continue + reponm = repo.attrib['desc'] + # This is only needed for the subprocess workaround. + cmd = ['yum', '-q', '-y', + # '--disablerepo=*', + '--enablerepo={0}'.format(repo.attrib['name'])] + pkgs = {'new': [], + 'upgrade': [], + 'downgrade': []} + for pkg in repo.findall('package'): + pkg_found = False + is_installed = False + if self.latest: + pkgnm = pkg.attrib['name'] + else: + pkgnm = pkg.attrib['NEVRA'] + pkglist = self.yb.doPackageLists(patterns = [pkgnm], showdups = True) + if pkglist.updates: + for pkgobj in reversed(pkglist.updates): + if pkgobj.repo.name == reponm: + # Haven't gotten this working properly. Patches welcome. + # self.yb.install(po = pkgobj) + # self.yb.resolveDeps() + # self.yb.buildTransaction() + # self.yb.processTransaction() + if self.latest: + pkgs['upgrade'].append(pkgobj.name) + else: + pkgs['upgrade'].append(pkgobj.nevra) + pkg_found = True + is_installed = False + break + if pkglist.installed and not pkg_found: + for pkgobj in reversed(pkglist.installed): + if pkgobj.repo.name == reponm: + warn = ('{0} from {1} is already installed; skipping').format(pkgobj.nevra, + repo.attrib['name']) + warnings.warn(warn) + pkg_found = True + is_installed = True + if not all((is_installed, pkg_found)): + if pkglist.available: + for pkgobj in reversed(pkglist.available): + if pkgobj.repo.name == reponm: + # Haven't gotten this working properly. Patches welcome. + # self.yb.install(po = pkgobj) + # self.yb.resolveDeps() + # self.yb.buildTransaction() + # self.yb.processTransaction() + if self.latest: + pkgs['new'].append(pkgobj.name) + else: + pkgs['new'].append(pkgobj.nevra) + is_installed = False + pkg_found = True + break + if not self.latest: + if pkglist.old_available: + for pkgobj in reversed(pkglist.old_available): + if pkgobj.repo.name == reponm: + # Haven't gotten this working properly. Patches welcome. + # self.yb.install(po = pkgobj) + # self.yb.resolveDeps() + # self.yb.buildTransaction() + # self.yb.processTransaction() + pkgs['downgrade'].append(pkgobj.nevra) + pkg_found = True + break + # # This... seems to always fail. Patches welcome. + # # self.yb.processTransaction() + for k in pkgs: + if not pkgs[k]: + continue + if pyver < py3: + _cmd = copy.deepcopy(cmd) + else: + _cmd = cmd.copy() + if k == 'downgrade': + _cmd.append('downgrade') + else: + if self.latest: + _cmd.append('install') + else: + if distver[0] >= 7: + _cmd.append('install-nevra') + else: + _cmd.append('install') + _cmd.extend(pkgs[k]) + if pyver >= py3: + subprocess.run(_cmd) + else: + subprocess.call(_cmd) + return() + + +def parseArgs(): + args = argparse.ArgumentParser(description = ('Reinstall packages from a generated XML package list')) + args.add_argument('-V', '--version', + dest = 'latest', + action = 'store_false', + help = ('If specified, (try to) install the same version as specified in the package list.')) + args.add_argument('pkglist_path', + metavar = 'PKGLIST', + help = ('The path to the generated packages XML file.')) + return(args) + +def main(): + args = parseArgs().parse_args() + dictargs = vars(args) + r = Reinstaller(**dictargs) + r.iterPkgs() + return() + +if __name__ == '__main__': + main() \ No newline at end of file