Automated commit (/opt/dev/infra/gitclass.py)

master
brent s. 2 years ago
parent d751d14be4
commit 803fb7c5fa
  1. 1
      README
  2. 21
      example.config.xml
  3. 64
      repomirror/sync.py

@ -10,7 +10,6 @@ Features:
* Changing of directory/file ownership per-distribution
* Mount-checking per-distribution - a distribution will be skipped if its designated mountpoint is not mounted
* Synchronization checks - timestamp files can be read and written and are used to determine if a sync should take place or not
** TODO: customizable "staleness" of repos (e.g. sync if timestamp X is older than timestamp Y on server)


Configuration/Deployment:

@ -43,18 +43,22 @@
<lastLocalSync timeFormat="UNIX_EPOCH">/srv/repos/arch/lastsync</lastLocalSync>
<!--
The path to a file on the upstream(s) that gives a time when it last updated.
The optional timeFormat attribute behavior is the same as above.
The syntax and options are the same as lastLocalCheck/lastLocalSync.
If neither this nor lastRemoteSync is provided, a sync will be attempted regardless of when the last one was
attempted.
-->
<!--
Remote timestamps take an additional optional boolean attribute, "mtime". If true, the mtime of the remote file
will be checked instead of the content of the file (and thus timeFormat is ignored).
-->
<lastRemoteUpdate timeFormat="UNIX_EPOCH">/lastupdate</lastRemoteUpdate>
<!--
The path to a file on the upstream(s) that gives a time when it last synced from its upstream.
The optional timeFormat attribute behavior is the same as above.
The syntax and options are the same as lastRemoteCheck.
If neither this nor lastRemoteUpdate is provided, a sync will be attempted regardless of when the last one was
attempted.
attempted. It follows the same rules as lastRemoteUpdate for syntax.
-->
<lastRemoteSync timeFormat="UNIX_EPOCH">/lastsync</lastRemoteSync>
<lastRemoteSync mtime="true" timeFormat="UNIX_EPOCH">/lastsync</lastRemoteSync>
<!--
The path that must be currently mounted for sync to proceed.
This is required.
@ -113,7 +117,14 @@
It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
-->
<upstream delayCheck="P0Y0M2DT0H0M0S">
<!--
You can optionally specify an offset via the "offset" attribute in the same format as "delayCheck" if your
upstream's remote files are using a different timezone instead of UTC.
e.g.:
* If your upstream uses UTC-4 for its timestamp files', you would use "-PT4H".
* If your upstream uses UTC+6 for its timestamp files, you would use either "+PT4H" or just "PT4H".
-->
<upstream delayCheck="P0Y0M2DT0H0M0S" offset="-PT0S">
<!--
The following example uses "rsync://arch.mirror.constant.com/archlinux/"
(https://www.archlinux.org/mirrors/constant.com/1008/)

@ -24,7 +24,7 @@ if os.isatty(sys.stdin.fileno()):
else:
_is_cron = True

_duration_re = re.compile(('^P'
_duration_re = re.compile(('^(?P<mod>[-+])?P'
'((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
'((?P<months>[0-9]+(\.[0-9]+)?)M)?'
'((?P<days>[0-9]+(\.[0-9]+)?)D)?'
@ -57,6 +57,19 @@ def get_owner(owner_xml):
return(owner)


def get_duration(duration_str):
r = _duration_re.search(duration_str)
times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
mod = times.pop('mod')
if not mod:
mod = '+'
years = float(times.pop('years'))
months = float(times.pop('months'))
times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
delay = datetime.timedelta(**times)
return((mod, delay))


class Args(object):
def __init__(self, args_xml):
self.xml = args_xml
@ -104,12 +117,15 @@ class Mount(object):

class TimestampFile(object):
def __init__(self, ts_xml, owner_xml = None):
self.xml = ts_xml
self.fmt = ts_xml.attrib.get('timeFormat', 'UNIX_EPOCH')
if self.fmt == 'UNIX_EPOCH':
self.fmt = '%s'
elif self.fmt == 'MICROSECOND_EPOCH':
self.fmt = '%s.%f'
_logger.debug('Set timestamp format string to {0}'.format(self.fmt))
self.mtime = (True if self.xml.attrib.get('mtime', 'false').lower().startswith(('t', '1')) else False)
_logger.debug('Using mtime: {0}'.format(self.mtime))
self.owner_xml = owner_xml
self.owner = {}
if self.owner_xml is not None:
@ -126,13 +142,16 @@ class TimestampFile(object):
else:
path = self.path
if os.path.isfile(path):
with open(path, 'r') as fh:
ts_raw = fh.read().strip()
if '%s' in self.fmt:
timestamp = datetime.datetime.fromtimestamp(float(ts_raw))
if self.mtime:
timestamp = datetime.datetime.fromtimestamp(float(os.stat(path).st_mtime))
else:
timestamp = datetime.datetime.strptime(ts_raw, self.fmt)
_logger.debug('Read timestamp {0} from {1}'.format(str(timestamp), self.path))
with open(path, 'r') as fh:
ts_raw = fh.read().strip()
if '%s' in self.fmt:
timestamp = datetime.datetime.fromtimestamp(float(ts_raw))
else:
timestamp = datetime.datetime.strptime(ts_raw, self.fmt)
_logger.debug('Read timestamp {0} from {1}'.format(str(timestamp), self.path))
return(timestamp)

def write(self):
@ -148,6 +167,9 @@ class TimestampFile(object):
os.chmod(self.path, mode = 0o0644)
if self.owner:
os.chown(self.path, **self.owner)
if self.mtime:
now = float(datetime.datetime.utcnow().timestamp())
os.utime(self.path, (now, now))
_logger.debug('Wrote timestamp to {0}'.format(self.path))
return(None)

@ -161,9 +183,11 @@ class Upstream(object):
self.path = self.xml.find('path').text
self.dest = os.path.abspath(os.path.expanduser(dest))
self.delay = None
self.offset = None
self.owner = owner
self.filechecks = filechecks
self._get_delaychk()
self._get_offset()
self.has_new = False
# These are optional.
port = self.xml.find('port')
@ -185,18 +209,6 @@ class Upstream(object):
self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
self._check_conn()

def _get_delaychk(self):
delay = self.xml.attrib.get('delayCheck')
if not delay:
return(None)
r = _duration_re.search(delay)
times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
years = float(times.pop('years'))
months = float(times.pop('months'))
times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
self.delay = datetime.timedelta(**times)
return(None)

def _check_conn(self):
sock = socket.socket()
sock.settimeout(7)
@ -208,6 +220,20 @@ class Upstream(object):
self.available = False
return(None)

def _get_delaychk(self):
delay = self.xml.attrib.get('delayCheck')
if not delay:
return(None)
mod, self.delay = get_duration(delay)
return(None)

def _get_offset(self):
offset = self.xml.attrib.get('offset')
if not offset:
return(None)
self.offset = get_duration(offset)
return(None)

def sync(self):
self.fetcher.fetch()
return(None)

Loading…
Cancel
Save