From 4bdb31541b324c5a6896bedf6c8413414b2b9193 Mon Sep 17 00:00:00 2001
From: brent s <r00t@square-r00t.net>
Date: Thu, 23 Jul 2020 12:05:36 -0400
Subject: [PATCH] adding in some delay checking calculation and parsing of
 xs:duration types.

---
 example.config.xml      |  9 ++++++-
 repomirror/constants.py | 19 +++++++++++---
 repomirror/sync.py      | 26 +++++++++++++++++++
 reposync                | 57 +----------------------------------------
 reposync.py             | 57 ++++++++++++++++++++++++++++++++++++++++-
 5 files changed, 106 insertions(+), 62 deletions(-)
 mode change 100755 => 120000 reposync
 mode change 120000 => 100755 reposync.py

diff --git a/example.config.xml b/example.config.xml
index 9301809..1451cfe 100644
--- a/example.config.xml
+++ b/example.config.xml
@@ -95,7 +95,14 @@
       -->
       <short>c</short><!-- checksum -->
     </rsyncArgs>
-    <upstream>
+    <!--
+      Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
+      https://en.wikipedia.org/wiki/ISO_8601#Durations
+      e.g. 5 minutes would be P0Y0M0DT0H5M0S or just PT5M.
+      It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
+      than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
+    -->
+    <upstream delayCheck="P0Y0M2DT0H0M0S">
       <!--
         The following example uses "rsync://arch.mirror.constant.com/archlinux/"
         (https://www.archlinux.org/mirrors/constant.com/1008/)
diff --git a/repomirror/constants.py b/repomirror/constants.py
index d11525a..207ab87 100644
--- a/repomirror/constants.py
+++ b/repomirror/constants.py
@@ -1,5 +1,6 @@
 PROTO_DEF_PORTS = {'ftp': 21,
                    'rsync': 873}
+
 RSYNC_DEF_ARGS = ['--recursive',
                   '--times',
                   '--links',
@@ -10,7 +11,17 @@ RSYNC_DEF_ARGS = ['--recursive',
                   '--safe-links',
                   '--delete-excluded',
                   '--exclude=.*']
-# How many days an upstream should have last synced by before it's considered stale.
-## TODO: make this part of the upstream config? repo config?
-DAYS_WARN = 2
-VERSION = '1.0.4'
+
+# These are needed to convert years/months to timedeltas.
+# The following are averaged definitions for time units *in days* according to Google Calculator.
+YEAR = 365.2422
+MONTH = 30.4167
+# The following are approximations based on ISO 8601 defintions *in days*.
+# https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm
+# YEAR = 365.25
+# MONTH = 30.6
+
+# We no longer do this by default.
+# # How many days an upstream should have last synced by before it's considered stale.
+# DAYS_WARN = 2
+VERSION = '1.1.0'
diff --git a/repomirror/sync.py b/repomirror/sync.py
index 3e9b16b..0226b3a 100644
--- a/repomirror/sync.py
+++ b/repomirror/sync.py
@@ -3,6 +3,7 @@ import logging
 import pwd
 import grp
 import os
+import re
 import socket
 import sys
 import warnings
@@ -23,6 +24,17 @@ if os.isatty(sys.stdin.fileno()):
 else:
     _is_cron = True
 
+_duration_re = re.compile(('^P'
+                           '((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
+                           '((?P<months>[0-9]+(\.[0-9]+)?)M)?'
+                           '((?P<days>[0-9]+(\.[0-9]+)?)D)?'
+                           'T?'
+                           '((?P<hours>[0-9]+(\.[0-9]+)?)H)?'
+                           '((?P<minutes>[0-9]+(\.[0-9]+)?)M)?'
+                           '((?P<seconds>[0-9]+(\.[0-9]+)?)S)?'
+                           '$'))
+
+
 
 def get_owner(owner_xml):
     owner = {}
@@ -149,8 +161,10 @@ class Upstream(object):
         self.domain = self.xml.find('domain').text
         self.path = self.xml.find('path').text
         self.dest = os.path.abspath(os.path.expanduser(dest))
+        self.delay = None
         self.owner = owner
         self.filechecks = filechecks
+        self._get_delaychk()
         self.has_new = False
         # These are optional.
         port = self.xml.find('port')
@@ -171,6 +185,18 @@ class Upstream(object):
             self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
         self._check_conn()
 
+    def _get_delaychk(self):
+        delay = self.xml.attrib.get('delayCheck')
+        if not delay:
+            return(None)
+        r = _duration_re.search(delay)
+        times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
+        years = float(times.pop('years'))
+        months = float(times.pop('months'))
+        times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
+        self.delay = datetime.timedelta(**times)
+        return(None)
+
     def _check_conn(self):
         sock = socket.socket()
         sock.settimeout(7)
diff --git a/reposync b/reposync
deleted file mode 100755
index a78bde5..0000000
--- a/reposync
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import os
-import pathlib
-import sys
-##
-##
-_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
-_cur_path = os.path.dirname(_cur_file)
-sys.path.insert(1, _cur_path)
-import repomirror
-
-
-if os.geteuid() == 0:
-    _def_logdir = '/var/log/repo'
-else:
-    _def_logdir = '~/.cache/repologs'
-
-
-def parseArgs():
-    args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
-    args.add_argument('-c', '--config',
-                      default = '~/.config/repomirror.xml',
-                      dest = 'cfg',
-                      help = ('The path to the config file. If it does not exist, a bare version will be created. '
-                              'Default: ~/.config/repomirror.xml'))
-    # args.add_argument('-n', '--dry-run',
-    #                   action = 'store_true',
-    #                   dest = 'dummy',
-    #                   help = ('If specified, do not actually sync anything (other than timestamp files if '
-    #                           'applicable to determine logic); do not actually sync any repositories. Useful for '
-    #                           'generating logs to determine potential issues before they happen'))
-    args.add_argument('-d', '--distro',
-                      dest = 'distro',
-                      action = 'append',
-                      help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
-                              'specified). May be given multiple times'))
-    args.add_argument('-l', '--logdir',
-                      default = _def_logdir,
-                      dest = 'logdir',
-                      help = ('The path to the directory where logs should be written. The actual log files will be '
-                              'named after their respective distro names in the config file. '
-                              'Default: {0}'.format(_def_logdir)))
-    return(args)
-
-
-def main():
-    args = parseArgs().parse_args()
-    r = repomirror.Sync(**vars(args))
-    r.sync()
-    return(None)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/reposync b/reposync
new file mode 120000
index 0000000..97ffdba
--- /dev/null
+++ b/reposync
@@ -0,0 +1 @@
+reposync.py
\ No newline at end of file
diff --git a/reposync.py b/reposync.py
deleted file mode 120000
index 9b00562..0000000
--- a/reposync.py
+++ /dev/null
@@ -1 +0,0 @@
-reposync
\ No newline at end of file
diff --git a/reposync.py b/reposync.py
new file mode 100755
index 0000000..a78bde5
--- /dev/null
+++ b/reposync.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+import sys
+##
+##
+_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
+_cur_path = os.path.dirname(_cur_file)
+sys.path.insert(1, _cur_path)
+import repomirror
+
+
+if os.geteuid() == 0:
+    _def_logdir = '/var/log/repo'
+else:
+    _def_logdir = '~/.cache/repologs'
+
+
+def parseArgs():
+    args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
+    args.add_argument('-c', '--config',
+                      default = '~/.config/repomirror.xml',
+                      dest = 'cfg',
+                      help = ('The path to the config file. If it does not exist, a bare version will be created. '
+                              'Default: ~/.config/repomirror.xml'))
+    # args.add_argument('-n', '--dry-run',
+    #                   action = 'store_true',
+    #                   dest = 'dummy',
+    #                   help = ('If specified, do not actually sync anything (other than timestamp files if '
+    #                           'applicable to determine logic); do not actually sync any repositories. Useful for '
+    #                           'generating logs to determine potential issues before they happen'))
+    args.add_argument('-d', '--distro',
+                      dest = 'distro',
+                      action = 'append',
+                      help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
+                              'specified). May be given multiple times'))
+    args.add_argument('-l', '--logdir',
+                      default = _def_logdir,
+                      dest = 'logdir',
+                      help = ('The path to the directory where logs should be written. The actual log files will be '
+                              'named after their respective distro names in the config file. '
+                              'Default: {0}'.format(_def_logdir)))
+    return(args)
+
+
+def main():
+    args = parseArgs().parse_args()
+    r = repomirror.Sync(**vars(args))
+    r.sync()
+    return(None)
+
+
+if __name__ == '__main__':
+    main()