updating to support configurable delay checking and rsync exit code ignoring

example.config.xml

@@ -69,16 +69,15 @@
           <long>links</long>
           <long>hard-links</long>
           <long>delete-after</long>
+          <long>perms</long>
           <long>delay-updates</long>
-          <long>copy-links</long>
           <long>safe-links</long>
           <long>delete-excluded</long>
-          <long value=".*">exclude</long>
         </rsyncArgs>
-      These arguments should be sane for most, if not all, rsync-driven repository mirroring. The last one (exclude) may
+      These arguments should be sane for most, if not all, rsync-driven repository mirroring.
-      be removed in future versions.
       If arguments are provided, the defaults are overwritten so if you need the above, be sure to specify them.
-      See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system.
+      See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system
+      (§ "OPTION SUMMARY", § "OPTIONS").
     -->
     <rsyncArgs>
       <!--
@@ -94,7 +93,19 @@
         A "short" option (single hyphen).
       -->
       <short>c</short><!-- checksum -->
+      <!--
+        An option that requires to be enclosed in quotes. (This one excludes hidden files/directories.)
+      -->
+      <long value="&quot;.*&quot;">exclude</long>
     </rsyncArgs>
+    <!--
+      rsyncIgnore is an optional element that ignores certain return codes/exit statuses of rsync (otherwise they are
+      raised as warnings). It consists of a space-separated list of return codes that warnings should be suppressed for.
+      (Return code 0 never raises a warning, as it is success.)
+      See the rsync man page (rsync(1)) for a list of error codes and what they mean (§ "EXIT VALUES"), or refer to:
+        repomirror/fetcher/rsync_returns.py
+    -->
+    <rsyncIgnore returns="23 24"/>
     <!--
       Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
       https://en.wikipedia.org/wiki/ISO_8601#Durations

repomirror/constants.py

@@ -6,11 +6,11 @@ RSYNC_DEF_ARGS = ['--recursive',
                   '--links',
                   '--hard-links',
                   '--delete-after',
+                  '--perms',
                   '--delay-updates',
-                  '--copy-links',
                   '--safe-links',
                   '--delete-excluded',
-                  '--exclude=.*']
+                  '--exclude=".*"']
 
 # These are needed to convert years/months to timedeltas.
 # The following are averaged definitions for time units *in days* according to Google Calculator.

repomirror/fetcher/rsync.py

@@ -25,6 +25,7 @@ class RSync(_base.BaseFetcher):
                  path,
                  dest,
                  rsync_args = None,
+                 rsync_ignores = None,
                  owner = None,
                  log = True,
                  filechecks = None,
@@ -37,6 +38,7 @@ class RSync(_base.BaseFetcher):
         else:
             self.rsync_args = constants.RSYNC_DEF_ARGS[:]
         _logger.debug('RSync args given: {0}'.format(self.rsync_args))
+        self.rsync_ignores = rsync_ignores[:]
         if log:
             # Do I want to do this in subprocess + logging module? Or keep this?
             # It looks a little ugly in the log but it makes more sense than doing it via subprocess just to write it
@@ -68,10 +70,10 @@ class RSync(_base.BaseFetcher):
                              stderr = subprocess.PIPE)
         stdout = cmd.stdout.decode('utf-8').strip()
         stderr = cmd.stderr.decode('utf-8').strip()
+        rtrn = cmd.returncode
         if stdout != '':
             _logger.debug('STDOUT: {0}'.format(stdout))
-        if stderr != '' or cmd.returncode != 0:
+        if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
-            rtrn = cmd.returncode
             err = rsync_returns.returns[rtrn]
             errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
             debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
@@ -100,10 +102,10 @@ class RSync(_base.BaseFetcher):
                              stderr = subprocess.PIPE)
         stdout = cmd.stdout.decode('utf-8').strip()
         stderr = cmd.stderr.decode('utf-8').strip()
+        rtrn = cmd.returncode
         if stdout != '':
             _logger.debug('STDOUT: {0}'.format(stdout))
-        if stderr != '' or cmd.returncode != 0:
+        if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
-            rtrn = cmd.returncode
             err = rsync_returns.returns[rtrn]
             errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
             debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))

repomirror/sync.py

@@ -35,7 +35,6 @@ _duration_re = re.compile(('^P'
                            '$'))
 
 
-
 def get_owner(owner_xml):
     owner = {}
     user = owner_xml.find('user')
@@ -154,7 +153,7 @@ class TimestampFile(object):
 
 
 class Upstream(object):
-    def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None):
+    def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None, rsync_ignores = None):
         self.xml = upstream_xml
         # These are required for all upstreams.
         self.sync_type = self.xml.find('syncType').text.lower()
@@ -179,6 +178,7 @@ class Upstream(object):
                                          self.path,
                                          self.dest,
                                          rsync_args = rsync_args,
+                                         rsync_ignores = rsync_ignores,
                                          filechecks = self.filechecks,
                                          owner = self.owner)
         else:
@@ -225,6 +225,7 @@ class Distro(object):
                                       'sync': None}}
         self.timestamps = {}
         self.rsync_args = None
+        self.rsync_ignores = None
         self.owner = None
         self.upstreams = []
         self.lockfile = '/var/run/repomirror/{0}.lck'.format(self.name)
@@ -243,12 +244,17 @@ class Distro(object):
             e = self.xml.find('lastRemote{0}'.format(i))
             if e is not None:
                 self.filechecks['remote'][i.lower()] = TimestampFile(e)
+        self.rsync_ignores = []
+        rsyncig_xml = self.xml.find('rsyncIgnore')
+        if rsyncig_xml is not None:
+            self.rsync_ignores = [int(i.strip()) for i in rsyncig_xml.attrib['returns'].split()]
         for u in self.xml.findall('upstream'):
             self.upstreams.append(Upstream(u,
                                            self.dest,
                                            rsync_args = self.rsync_args,
                                            owner = self.owner,
-                                           filechecks = self.filechecks))
+                                           filechecks = self.filechecks,
+                                           rsync_ignores = self.rsync_ignores))
 
     def check(self):
         for k, v in self.filechecks['local'].items():
@@ -285,11 +291,11 @@ class Distro(object):
                 else:
                     _logger.info('No remote update timestamp; syncing.')
                     u.has_new = True
-                if sync:
+                if sync and u.delay:
                     td = datetime.datetime.utcnow() - sync
-                    if td.days > constants.DAYS_WARN:
+                    if td.days > u.delay:
-                        _logger.warning(('Upstream {0} has not synced for {1} or more days; this '
+                        _logger.warning(('Upstream {0} has not synced for {1} or longer; this '
-                                         'repository may be out of date.').format(u.fetcher.url, constants.DAYS_WARN))
+                                         'repository may be out of date.').format(u.fetcher.url, u.delay))
                         warnings.warn('Upstream may be out of date')
         return(None)