[wip] Repo pruning

data/actionsmap/yunohost.yml

@@ -1041,7 +1041,7 @@ backup:
                     help: Name of the backup archive
                     extra:
                         pattern: &pattern_backup_archive_name
-                            - !!str ^[\w\-\._]{1,50}(?<!\.)$
+                            - !!str ^[\w\-\._]{1,50}$
                             - "pattern_backup_archive_name"
                 -d:
                     full: --description
@@ -1255,9 +1255,9 @@ backup:
                         name:
                             help: Short prefix of the backup archives
                             extra:
-                                pattern: &pattern_backup_archive_name
+                                pattern: &pattern_backup_timer_name
-                                    - !!str ^[\w\-\._]{1,50}(?<!\.)$
+                                    - !!str ^[\w\-\._]{1,50}$
-                                    - "pattern_backup_archive_name"
+                                    - "pattern_backup_timer_name"
                         -d:
                             full: --description
                             help: Short description of the backup
@@ -1291,6 +1291,8 @@ backup:
                     arguments:
                         name:
                             help: Short prefix of the backup archives
+                            extra:
+                                pattern: *pattern_backup_timer_name
                         -d:
                             full: --description
                             help: Short description of the backup
@@ -1324,6 +1326,8 @@ backup:
                     arguments:
                         name:
                             help: Short prefix of the backup archives
+                            extra:
+                                pattern: *pattern_backup_timer_name
 
                 ### backup_timer_info()
                 info:
@@ -1332,6 +1336,8 @@ backup:
                     arguments:
                         name:
                             help: Short prefix of the backup archives
+                            extra:
+                                pattern: *pattern_backup_timer_name
 
 #############################
 #         Settings          #

src/yunohost/repositories/borg.py

@@ -18,6 +18,18 @@
     along with this program; if not, see http://www.gnu.org/licenses
 
 """
+import os
+import subprocess
+import json
+
+from datetime import datetime
+
+from moulinette.utils.log import getActionLogger
+from moulinette import m18n
+
+from yunohost.utils.error import YunohostError
+from yunohost.repository import LocalBackupRepository
+logger = getActionLogger("yunohost.repository")
 
 
 class BorgBackupRepository(LocalBackupRepository):
@@ -59,7 +71,7 @@ class BorgBackupRepository(LocalBackupRepository):
         if json_output:
             try:
                 return json.loads(out)
-            except json.decoder.JSONDecodeError, TypeError:
+            except (json.decoder.JSONDecodeError, TypeError):
                 raise YunohostError(f"backup_borg_{action}_error")
         return out
 
@@ -79,12 +91,12 @@ class BorgBackupRepository(LocalBackupRepository):
                     domain=self.domain,
                     service=services[self.method],
                     shf_id=values.pop('shf_id', None),
-                    data = {
+                    data={
                         'origin': self.domain,
                         'public_key': self.public_key,
-                        'quota':  self.quota,
+                        'quota': self.quota,
-                        'alert':  self.alert,
+                        'alert': self.alert,
-                        'alert_delay':  self.alert_delay,
+                        'alert_delay': self.alert_delay,
                         #password: "XXXXXXXX",
                     }
                 )
@@ -99,7 +111,6 @@ class BorgBackupRepository(LocalBackupRepository):
         else:
             super().install()
 
-
         # Initialize borg repo
         cmd = ["borg", "init", "--encryption", "repokey", self.location]
 
@@ -107,7 +118,6 @@ class BorgBackupRepository(LocalBackupRepository):
             cmd += ['--storage-quota', self.quota]
         self._call('init', cmd)
 
-
     def update(self):
         raise NotImplementedError()
 
@@ -117,7 +127,7 @@ class BorgBackupRepository(LocalBackupRepository):
                 domain=self.domain,
                 service="borgbackup",
                 shf_id=values.pop('shf_id', None),
-                data = {
+                data={
                     'origin': self.domain,
                     #password: "XXXXXXXX",
                 }
@@ -143,7 +153,7 @@ class BorgBackupRepository(LocalBackupRepository):
             response = self._call('info', cmd)
             return response["cache"]["stats"]["unique_size"]
 
-    def prune(self, prefix=None, hourly=None, daily=None, weekly=None, yearly=None):
+    def prune(self, prefix=None, **kwargs):
 
         # List archives with creation date
         archives = {}
@@ -155,17 +165,30 @@ class BorgBackupRepository(LocalBackupRepository):
         if not archives:
             return
 
-        # Generate period in which keep one archive
+        # Generate periods in which keep one archive
-        now = datetime.now()
+        now = datetime.utcnow()
-        periods = []
+        now -= timedelta(
-        for in range(hourly):
+            minutes=now.minute,
-
+            seconds=now.second,
+            microseconds=now.microsecond
+        )
+        periods = set([])
 
+        for unit, qty in kwargs:
+            if not qty:
+                continue
+            period = timedelta(**{unit: 1})
+            periods += set([(now - period * i, now - period * (i - 1))
+                        for i in range(qty)])
 
+        # Delete unneeded archive
         for created_at in sorted(archives, reverse=True):
-            created_at = datetime.fromtimestamp(created_at)
+            created_at = datetime.utcfromtimestamp(created_at)
-            if hourly > 0 and :
+            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
-                hourly-=1
+
+            if keep_for:
+                periods -= keep_for
+                continue
 
             archive.delete()
 

src/yunohost/repositories/tar.py

@@ -18,11 +18,26 @@
     along with this program; if not, see http://www.gnu.org/licenses
 
 """
+import glob
+import os
+import tarfile
+import shutil
+
+from moulinette.utils.log import getActionLogger
+from moulinette import m18n
+
+from yunohost.utils.error import YunohostError, YunohostValidationError
+from yunohost.repository import LocalBackupRepository
+from yunohost.backup import BackupManager
+from yunohost.utils.filesystem import space_used_in_directory
+from yunohost.settings import settings_get
+logger = getActionLogger("yunohost.repository")
 
 
 class TarBackupRepository(LocalBackupRepository):
     need_organized_files = False
     method_name = "tar"
+
     def list_archives_names(self):
         # Get local archives sorted according to last modification time
         # (we do a realpath() to resolve symlinks)
@@ -96,7 +111,7 @@ class TarBackupArchive:
         # Move info file
         shutil.copy(
             os.path.join(self.work_dir, "info.json"),
-            os.path.join(ARCHIVES_PATH, self.name + ".info.json"),
+            os.path.join(self.repo.location, self.name + ".info.json"),
         )
 
         # If backuped to a non-default location, keep a symlink of the archive

src/yunohost/repository.py

@@ -58,6 +58,7 @@ REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
 # TODO F2F server
 # TODO i18n pattern error
 
+
 class BackupRepository(ConfigPanel):
     """
     BackupRepository manage all repository the admin added to the instance
@@ -74,7 +75,7 @@ class BackupRepository(ConfigPanel):
         Split a repository location into protocol, user, domain and path
         """
         if "/" not in location:
-            return { "domain": location }
+            return {"domain": location}
 
         location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+):((?P<port>\d+)/)?)?(?P<path>[^:]+)$'
         location_match = re.match(location_regex, location)
@@ -198,10 +199,13 @@ class BackupRepository(ConfigPanel):
 
         if self.__class__ == BackupRepository:
             if self.method == 'tar':
+                from yunohost.repositories.tar import TarBackupRepository
                 self.__class__ = TarBackupRepository
             elif self.method == 'borg':
+                from yunohost.repositories.borg import BorgBackupRepository
                 self.__class__ = BorgBackupRepository
             else:
+                from yunohost.repositories.hook import HookBackupRepository
                 self.__class__ = HookBackupRepository
 
     def _check_is_enough_free_space(self):
@@ -259,6 +263,45 @@ class BackupRepository(ConfigPanel):
 
         return archives
 
+    def prune(self, prefix=None, **kwargs):
+
+        # List archives with creation date
+        archives = {}
+        for archive_name in self.list_archive_name(prefix):
+            archive = BackupArchive(repo=self, name=archive_name)
+            created_at = archive.info()["created_at"]
+            archives[created_at] = archive
+
+        if not archives:
+            return
+
+        # Generate periods in which keep one archive
+        now = datetime.utcnow()
+        now -= timedelta(
+            minutes=now.minute,
+            seconds=now.second,
+            microseconds=now.microsecond
+        )
+        periods = set([])
+
+        for unit, qty in kwargs:
+            if not qty:
+                continue
+            period = timedelta(**{unit: 1})
+            periods += set([(now - period * i, now - period * (i - 1))
+                        for i in range(qty)])
+
+        # Delete unneeded archive
+        for created_at in sorted(archives, reverse=True):
+            created_at = datetime.utcfromtimestamp(created_at)
+            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
+
+            if keep_for:
+                periods -= keep_for
+                continue
+
+            archive.delete()
+
     # =================================================
     # Repository abstract actions
     # =================================================
@@ -277,8 +320,6 @@ class BackupRepository(ConfigPanel):
     def compute_space_used(self):
         raise NotImplementedError()
 
-    def prune(self):
-        raise NotImplementedError() # TODO prune
 
 class LocalBackupRepository(BackupRepository):
     def install(self):