From 8e7841b4cc2ecde5b02c21258585e2b0efae21d1 Mon Sep 17 00:00:00 2001
From: brent s <bts@square-r00t.net>
Date: Sun, 22 May 2022 20:18:05 -0400
Subject: [PATCH] fix. --keep-within and --keep-<period> are different options.

---
 backup.py                 | 52 +++++++++++++++++++++++++-------------
 config.xsd                | 23 ++++++++++++++---
 sample.config.snippet.xml |  5 +---
 sample.config.xml         | 53 ++++++++++++++++++++++++---------------
 4 files changed, 88 insertions(+), 45 deletions(-)

diff --git a/backup.py b/backup.py
index 674bae1..6a1d55d 100755
--- a/backup.py
+++ b/backup.py
@@ -45,6 +45,8 @@ loglvls = {'critical': logging.CRITICAL,
 ### DEFAULT NAMESPACE ###
 dflt_ns = 'http://git.root2.io/r00t2/borgextend/'
 
+# In code, replace "day" with "daily" when constructing command.
+policyperiods = ('Second', 'Minute', 'Hour', 'Day', 'Week', 'Month', 'Year')
 
 ### THE GUTS ###
 class Backup(object):
@@ -127,9 +129,6 @@ class Backup(object):
             if not reponames:
                 reponames = []
             repos = []
-            dfltRetention = None
-            if server.attrib.get('pruneRetention') is not None:
-                dfltRetention = isodate.parse_duration(server.attrib.get('pruneRetention'))
             for repo in server.findall('{0}repo'.format(self.ns)):
                 if reponames and repo.attrib['name'] not in reponames:
                     continue
@@ -140,8 +139,8 @@ class Backup(object):
                     r[a] = repo.attrib[a]
                 for e in ('path', 'exclude'):
                     # TODO: have an attrib for path and exclude, "glob=<bool>"?
-                    # If true, try using the glob module to resolve paths?
-                    # This gives us the benefit of allowing glob per-path/exclude.
+                    #       If true, try using the glob module to resolve paths?
+                    #       This gives us the benefit of allowing glob per-path/exclude.
                     r[e] = [i.text for i in repo.findall(self.ns + e)]
                 for prep in repo.findall('{0}prep'.format(self.ns)):
                     if 'prep' not in r:
@@ -164,11 +163,21 @@ class Backup(object):
                                 r['plugins'][pname]['params'][paramname] = json.loads(param.text)
                             else:
                                 r['plugins'][pname]['params'][paramname] = param.text
-                retention = repo.attrib.get('pruneRetention')
-                if retention is not None:
-                    r['retention'] = isodate.parse_duration(retention)
-                else:
-                    r['retention'] = dfltRetention
+                keepWithin = repo.find('{0}keepWithin'.format(self.ns))
+                if keepWithin is not None:
+                    if 'retention' not in r.keys():
+                        r['retention'] = {}
+                    r['retention']['last'] = isodate.parse_duration(keepWithin.text)
+                keepLast = repo.find('{0}keepLast'.format(self.ns))
+                if keepLast is not None:
+                    for e in policyperiods:
+                        k = e.lower()
+                        policy = keepLast.find('{0}per{1}'.format(self.ns, e))
+                        if policy is not None:
+                            if 'retention' not in r.keys():
+                                r['retention'] = {}
+                            # This is safe. We validate the config.
+                            r['retention'][k] = int(policy.text)
                 repos.append(r)
             return(repos)
         self.logger.debug('VARS (before args cleanup): {0}'.format(vars(self)))
@@ -482,12 +491,8 @@ class Backup(object):
             _user = self.repos[server].get('user', pwd.getpwuid(os.geteuid()).pw_name)
             for repo in self.repos[server]['repos']:
                 if repo.get('retention') is None:
-                    # No prune duration was set. Skip.
+                    # No prune retention was set. Skip.
                     continue
-                if isinstance(repo['retention'], datetime.timedelta):
-                    retentionSeconds = repo['retention'].total_seconds()
-                else:  # it's an isodate.Duration
-                    retentionSeconds = repo['retention'].totimedelta(datetime.datetime.now()).total_seconds()
                 _loc_env = _env.copy()
                 if 'password' not in repo:
                     print('Password not supplied for {0}:{1}.'.format(server, repo['name']))
@@ -500,8 +505,21 @@ class Backup(object):
                         '--log-json',
                         '--{0}'.format(self.args['loglevel']),
                         'prune',
-                        '--stats',
-                        '--keep-secondly', int(retentionSeconds)]
+                        '--stats']
+                # keepWithin
+                if repo['retention'].get('last') is not None:
+                    if isinstance(repo['retention'].get('last'), datetime.timedelta):  # it's a time.timedelta
+                        retentionSeconds = repo['retention'].total_seconds()
+                    else:  # it's an isodate.Duration
+                        retentionSeconds = repo['retention'].totimedelta(datetime.datetime.now()).total_seconds()
+                    _cmd.extend(['--keep-within',  '{0}H'.format(retentionSeconds / 60)])
+                # keepLast
+                for e in policyperiods:
+                    if repo['retention'].get(e.lower()) is not None:
+                        a = e.lower() + 'ly'
+                        if e.lower() == 'day':
+                            a = 'daily'
+                        _cmd.extend(['--keep-{0}'.format(a), repo['retention'][e.lower()]])
                 if self.repos[server]['remote'].lower()[0] in ('1', 't'):
                     repo_tgt = '{0}@{1}'.format(_user, server)
                 else:
diff --git a/config.xsd b/config.xsd
index f359bc1..f9cc0e1 100644
--- a/config.xsd
+++ b/config.xsd
@@ -43,6 +43,7 @@
     </xs:simpleContent>
   </xs:complexType>
 
+
   <!-- START ROOT -->
   <xs:element name="borg">
     <xs:complexType>
@@ -56,6 +57,24 @@
               <xs:element name="repo" minOccurs="1" maxOccurs="unbounded">
                 <xs:complexType>
                   <xs:choice minOccurs="1" maxOccurs="unbounded">
+                    <!-- START RETENTION POLICY -->
+                    <!-- This is used to specify the per-repo retention *period* for pruning. -->
+                    <xs:element name="keepWithin" minOccurs="0" maxOccurs="1" type="xs:duration"/>
+                    <!-- This is used to specify more advanced retentions. -->
+                    <xs:element name="keepLast" minOccurs="0" maxOccurs="1">
+                      <xs:complexType>
+                        <xs:choice minOccurs="1" maxOccurs="7">
+                          <xs:element name="perSecond" minOccurs="0" maxOccurs="1" type="xs:positiveInteger"/>
+                          <xs:element name="perMinute" minOccurs="0" maxOccurs="1" type="xs:positiveInteger"/>
+                          <xs:element name="perHour" minOccurs="0" maxOccurs="1" type="xs:positiveInteger"/>
+                          <xs:element name="perDay" minOccurs="0" maxOccurs="1" type="xs:positiveInteger"/>
+                          <xs:element name="perWeek" minOccurs="0" maxOccurs="1" type="xs:positiveInteger"/>
+                          <xs:element name="perMonth" minOccurs="0" maxOccurs="1" type="xs:positiveInteger"/>
+                          <xs:element name="perYear" minOccurs="0" maxOccurs="1" type="xs:positiveInteger"/>
+                        </xs:choice>
+                      </xs:complexType>
+                    </xs:element>
+                    <!-- END RETENTION POLICY -->
                     <!-- START PATH -->
                     <xs:element name="path" minOccurs="1"
                                 maxOccurs="unbounded" type="xs:anyURI"/>
@@ -100,8 +119,6 @@
                   <!-- This specifies if a repo is a "dummy" configuration.
                        Useful for testing and placeholder. -->
                   <xs:attribute name="dummy" type="xs:boolean" use="optional" default="false"/>
-                  <!-- This is used to specify the per-repo retention period for pruning. -->
-                  <xs:attribute name="pruneRetention" type="xs:duration" use="optional"/>
                 </xs:complexType>
                 <xs:unique name="uniquePath">
                   <xs:selector xpath="path"/>
@@ -124,8 +141,6 @@
             <!-- This specifies if a server is a "dummy" configuration.
                  Useful for testing and placeholder. -->
             <xs:attribute name="dummy" type="xs:boolean" use="optional" default="false"/>
-            <!-- This is used to specify the server-wide default retention period for pruning. -->
-            <xs:attribute name="pruneRetention" type="xs:duration" use="optional"/>
           </xs:complexType>
         </xs:element>
         <!-- END SERVER -->
diff --git a/sample.config.snippet.xml b/sample.config.snippet.xml
index e1710e1..e9cb309 100644
--- a/sample.config.snippet.xml
+++ b/sample.config.snippet.xml
@@ -1,8 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-        <repo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-              xmlns="http://git.root2.io/r00t2/borgextend/"
-              xsi:schemaLocation="http://git.root2.io/r00t2/borgextend/ http://git.r00t2.io/r00t2/borgextend/src/branch/master/config.xsd"
-              name="testrepo2"
+        <repo name="testrepo2"
               password="AnotherSuperSecretPassword"
               dummy="true">
             <path>/dev/null</path>
diff --git a/sample.config.xml b/sample.config.xml
index 7c8a1f8..86ce93d 100644
--- a/sample.config.xml
+++ b/sample.config.xml
@@ -12,17 +12,9 @@
         "dummy" = a boolean; if you need to create a "dummy" server, set this to "true".
                   It will *not* be parsed or executed upon.
                   It won't even be created by an init operation or show up in a repolist operation.
-        "pruneRetention" = an ISO 8601 duration to be used as the default rentention period during a prune operation.
     -->
-    <!--
-        The pruneRetention attribute's format (ISO 8601 Duration) can be found here:
-            https://en.wikipedia.org/wiki/ISO_8601#Durations).
-        It must be positive. The "alternative"/"extended" variant (the one using colons) should be supported
-        (but may be unpredictable).
-        It is optional, but can be used as a fallback during a prune operation if a repo child does not specify one.
-        The below example would prune anything older than 1 month, 2 minutes, and 30 seconds.
-    -->
-    <server target="fq.dn.tld" remote="true" rsh="ssh -p 22" user="root" pruneRetention="P1MT2M30S">
+
+    <server target="fq.dn.tld" remote="true" rsh="ssh -p 22" user="root">
         <!-- You can (and probably will) have multiple repos for each server. -->
         <!--
             "name" = the repository name.
@@ -30,20 +22,41 @@
                          to enter it interactively and securely.
             "dummy" = see server[@dummy] explanation.
             "compression" = see https://borgbackup.readthedocs.io/en/stable/usage/create.html (-C option)
-            "pruneRetention" = an ISO 8601 duration to be used as the rentention period during a prune operation.
-                               If not specified, uses ../server/[@pruneRetention].
-                               If ../server/[@pruneRetention] was not specified, this repo will be skipped during a prune.
         -->
         <repo name="testrepo"
               password="SuperSecretPassword"
-              compression="lzma,9"
-              pruneRetention="P1Y"><!-- One year. -->
-            <!-- Each path entry is a path to back up.
-                 See https://borgbackup.readthedocs.io/en/stable/usage/create.html
-                 Note that globbing, etc. is *disabled* for security reasons, so you will need to specify all
-                 directories explicitly. -->
+              compression="lzma,9">
+            <!--
+                keepWithin specifies a flat time period for archive retention during a prune.
+                The keepWithin attribute's format (ISO 8601 Duration) can be found here:
+                    https://en.wikipedia.org/wiki/ISO_8601#Durations).
+                It must be positive. The "alternative"/"extended" variant (the one using colons) should be supported
+                (but may be unpredictable).
+                It is optional. If no keepWithin or keepLast (below) is specified, this repo will be skipped during
+                prune operations.
+                The below example would prune anything older than 1 month, 2 minutes, and 30 seconds.
+                If both keepWithin and keepLast is provided, keepLast only affects archives *before* the time period
+                specified by keepWithin.
+            -->
+            <keepWithin>P1MT2M30S</keepWithin>
+            <!--
+                keepLast specifies a *policy* for retention.
+                It can be used to specify explicit "N archives per Y". For example, the below retains the first
+                archive for each month (going back 2 months), and the first archive each year for the last three years.
+            -->
+            <keepLast>
+                <perMonth>2</perMonth>
+                <perYear>3</perYear>
+            </keepLast>
+            <!--
+                Each path entry is a path to back up.
+                See https://borgbackup.readthedocs.io/en/stable/usage/create.html
+                Note that globbing, etc. is *disabled* for security reasons, so you will need to specify all
+                parent directories explicitly.
+                Recursing is enabled, though.
+            -->
             <path>/a</path>
-            <!-- Each exclude entry should be a subdirectory of a <path> (otherwise it wouldn't match, obviously). -->
+            <!-- Each exclude entry should be a subdirectory of a <path> (otherwise it wouldn't match). -->
             <exclude>/a/b</exclude>
             <!-- Prep items are executed in non-guaranteed order (but are likely to be performed in order given).
                  If you require them to be in a specific order, you should use a wrapper script and