adding in some delay checking calculation and parsing of xs:duration types.

master
brent s 2 years ago
parent a3203ab03a
commit 4bdb31541b
Signed by: bts
GPG Key ID: 8C004C2F93481F6B
  1. 9
      example.config.xml
  2. 19
      repomirror/constants.py
  3. 26
      repomirror/sync.py
  4. 56
      reposync
  5. 1
      reposync
  6. 1
      reposync.py
  7. 56
      reposync.py

@ -95,7 +95,14 @@
-->
<short>c</short><!-- checksum -->
</rsyncArgs>
<upstream>
<!--
Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
https://en.wikipedia.org/wiki/ISO_8601#Durations
e.g. 5 minutes would be P0Y0M0DT0H5M0S or just PT5M.
It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
-->
<upstream delayCheck="P0Y0M2DT0H0M0S">
<!--
The following example uses "rsync://arch.mirror.constant.com/archlinux/"
(https://www.archlinux.org/mirrors/constant.com/1008/)

@ -1,5 +1,6 @@
PROTO_DEF_PORTS = {'ftp': 21,
'rsync': 873}

RSYNC_DEF_ARGS = ['--recursive',
'--times',
'--links',
@ -10,7 +11,17 @@ RSYNC_DEF_ARGS = ['--recursive',
'--safe-links',
'--delete-excluded',
'--exclude=.*']
# How many days an upstream should have last synced by before it's considered stale.
## TODO: make this part of the upstream config? repo config?
DAYS_WARN = 2
VERSION = '1.0.4'

# These are needed to convert years/months to timedeltas.
# The following are averaged definitions for time units *in days* according to Google Calculator.
YEAR = 365.2422
MONTH = 30.4167
# The following are approximations based on ISO 8601 defintions *in days*.
# https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm
# YEAR = 365.25
# MONTH = 30.6

# We no longer do this by default.
# # How many days an upstream should have last synced by before it's considered stale.
# DAYS_WARN = 2
VERSION = '1.1.0'

@ -3,6 +3,7 @@ import logging
import pwd
import grp
import os
import re
import socket
import sys
import warnings
@ -23,6 +24,17 @@ if os.isatty(sys.stdin.fileno()):
else:
_is_cron = True

_duration_re = re.compile(('^P'
'((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
'((?P<months>[0-9]+(\.[0-9]+)?)M)?'
'((?P<days>[0-9]+(\.[0-9]+)?)D)?'
'T?'
'((?P<hours>[0-9]+(\.[0-9]+)?)H)?'
'((?P<minutes>[0-9]+(\.[0-9]+)?)M)?'
'((?P<seconds>[0-9]+(\.[0-9]+)?)S)?'
'$'))



def get_owner(owner_xml):
owner = {}
@ -149,8 +161,10 @@ class Upstream(object):
self.domain = self.xml.find('domain').text
self.path = self.xml.find('path').text
self.dest = os.path.abspath(os.path.expanduser(dest))
self.delay = None
self.owner = owner
self.filechecks = filechecks
self._get_delaychk()
self.has_new = False
# These are optional.
port = self.xml.find('port')
@ -171,6 +185,18 @@ class Upstream(object):
self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
self._check_conn()

def _get_delaychk(self):
delay = self.xml.attrib.get('delayCheck')
if not delay:
return(None)
r = _duration_re.search(delay)
times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
years = float(times.pop('years'))
months = float(times.pop('months'))
times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
self.delay = datetime.timedelta(**times)
return(None)

def _check_conn(self):
sock = socket.socket()
sock.settimeout(7)

@ -1,56 +0,0 @@
#!/usr/bin/env python3

import argparse
import os
import pathlib
import sys
##
##
_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
_cur_path = os.path.dirname(_cur_file)
sys.path.insert(1, _cur_path)
import repomirror


if os.geteuid() == 0:
_def_logdir = '/var/log/repo'
else:
_def_logdir = '~/.cache/repologs'


def parseArgs():
args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
args.add_argument('-c', '--config',
default = '~/.config/repomirror.xml',
dest = 'cfg',
help = ('The path to the config file. If it does not exist, a bare version will be created. '
'Default: ~/.config/repomirror.xml'))
# args.add_argument('-n', '--dry-run',
# action = 'store_true',
# dest = 'dummy',
# help = ('If specified, do not actually sync anything (other than timestamp files if '
# 'applicable to determine logic); do not actually sync any repositories. Useful for '
# 'generating logs to determine potential issues before they happen'))
args.add_argument('-d', '--distro',
dest = 'distro',
action = 'append',
help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
'specified). May be given multiple times'))
args.add_argument('-l', '--logdir',
default = _def_logdir,
dest = 'logdir',
help = ('The path to the directory where logs should be written. The actual log files will be '
'named after their respective distro names in the config file. '
'Default: {0}'.format(_def_logdir)))
return(args)


def main():
args = parseArgs().parse_args()
r = repomirror.Sync(**vars(args))
r.sync()
return(None)


if __name__ == '__main__':
main()

@ -0,0 +1 @@
reposync.py

@ -1 +0,0 @@
reposync

@ -0,0 +1,56 @@
#!/usr/bin/env python3

import argparse
import os
import pathlib
import sys
##
##
_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
_cur_path = os.path.dirname(_cur_file)
sys.path.insert(1, _cur_path)
import repomirror


if os.geteuid() == 0:
_def_logdir = '/var/log/repo'
else:
_def_logdir = '~/.cache/repologs'


def parseArgs():
args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
args.add_argument('-c', '--config',
default = '~/.config/repomirror.xml',
dest = 'cfg',
help = ('The path to the config file. If it does not exist, a bare version will be created. '
'Default: ~/.config/repomirror.xml'))
# args.add_argument('-n', '--dry-run',
# action = 'store_true',
# dest = 'dummy',
# help = ('If specified, do not actually sync anything (other than timestamp files if '
# 'applicable to determine logic); do not actually sync any repositories. Useful for '
# 'generating logs to determine potential issues before they happen'))
args.add_argument('-d', '--distro',
dest = 'distro',
action = 'append',
help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
'specified). May be given multiple times'))
args.add_argument('-l', '--logdir',
default = _def_logdir,
dest = 'logdir',
help = ('The path to the directory where logs should be written. The actual log files will be '
'named after their respective distro names in the config file. '
'Default: {0}'.format(_def_logdir)))
return(args)


def main():
args = parseArgs().parse_args()
r = repomirror.Sync(**vars(args))
r.sync()
return(None)


if __name__ == '__main__':
main()
Loading…
Cancel
Save