[enh] Prune and keep options

locales/en.json

@@ -114,6 +114,7 @@
     "backup_not_sent": "Backup archive was not saved at all",
     "backup_partially_sent": "Backup archive was not sent into all repositories listed",
     "backup_nothings_done": "Nothing to save",
+    "backup_nowhere_to_backup": "Nowhere to backup your file with this archive name",
     "backup_output_directory_forbidden": "Pick a different output directory. Backups cannot be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders",
     "backup_output_directory_not_empty": "You should pick an empty output directory",
     "backup_output_directory_required": "You must provide an output directory for the backup",

share/actionsmap.yml

@@ -1047,6 +1047,13 @@ backup:
                         pattern: &pattern_backup_archive_name
                             - !!str ^[\w\-\._]{1,50}$
                             - "pattern_backup_archive_name"
+                -p:
+                    full: --prefix
+                    help: Prefix of the backup archive
+                    extra:
+                        pattern: &pattern_backup_archive_prefix
+                            - !!str ^[\w\-\._]{1,35}$
+                            - "pattern_backup_archive_prefix"
                 -d:
                     full: --description
                     help: Short description of the backup
@@ -1277,6 +1284,33 @@ backup:
                         --prefix:
                             help: Prefix on which we prune
                             nargs: "?"
+                        -H:
+                            full: --keep-hourly
+                            help: Number of hourly archives to keep
+                            type: int
+                        -d:
+                            full: --keep-daily
+                            help: Number of daily archives to keep
+                            type: int
+                        -w:
+                            full: --keep-weekly
+                            help: Number of weekly archives to keep
+                            type: int
+                        -m:
+                            full: --keep-monthly
+                            help: Number of monthly archives to keep
+                            type: int
+                        --keep-last:
+                            help: Number of last archives to keep
+                            type: int
+                        --keep-within:
+                            help: Keep all archives within this time interval
+                            extra:
+                                pattern: &pattern_interval
+                                    - !!str ^\d+[Hdwmy]$
+                                    - "pattern_interval"
+
+
         timer:
             subcategory_help: Manage backup timer
             actions:

src/backup.py

@@ -25,6 +25,7 @@ import csv
 import tempfile
 import re
 import urllib.parse
+import datetime
 from packaging import version
 
 from moulinette import Moulinette, m18n
@@ -260,7 +261,7 @@ class BackupManager:
         backup_manager.backup()
     """
 
-    def __init__(self, name=None, description="", repositories=[], work_dir=None):
+    def __init__(self, name=None, prefix="", description="", repositories=[], work_dir=None):
         """
         BackupManager constructor
 
@@ -286,7 +287,7 @@ class BackupManager:
         self.targets = BackupRestoreTargetsManager()
 
         # Define backup name if needed
-
+        self.prefix = prefix
         if not name:
             name = self._define_backup_name()
         self.name = name
@@ -334,7 +335,7 @@ class BackupManager:
         """
         # FIXME: case where this name already exist
 
-        return time.strftime("%Y%m%d-%H%M%S", time.gmtime())
+        return self.prefix + time.strftime("%Y%m%d-%H%M%S", time.gmtime())
 
     def _init_work_dir(self):
         """Initialize preparation directory
@@ -1648,6 +1649,7 @@ class RestoreManager:
 def backup_create(
     operation_logger,
     name=None,
+    prefix="",
     description=None,
     repositories=[],
     system=[],
@@ -1679,13 +1681,15 @@ def backup_create(
 
     # Validate there is no archive with the same name
     archives = backup_list(repositories=repositories)
+    archives_already_exists = []
     for repository in archives:
         if name and name in archives[repository]:
-            repositories.pop(repository)
+            repositories.remove(repository)
+            archives_already_exists.append(repository)
             logger.error(m18n.n("backup_archive_name_exists", repository=repository))
 
     if not repositories:
-        raise YunohostValidationError("backup_archive_name_exists")
+        raise YunohostValidationError("backup_nowhere_to_backup")
 
     # If no --system or --apps given, backup everything
 
@@ -1702,7 +1706,8 @@ def backup_create(
     repositories = [BackupRepository(repo) for repo in repositories]
 
     # Prepare files to backup
-    backup_manager = BackupManager(name, description,
+    backup_manager = BackupManager(name, prefix=prefix,
+                                   description=description,
                                    repositories=repositories)
 
     # Add backup targets (system and apps)
@@ -1741,6 +1746,7 @@ def backup_create(
         )
     )
     repo_results = backup_manager.backup()
+    repo_results.update({repo: "Not sent" for repo in archives_already_exists})
     repo_states = [repo_result == "Sent" for repository, repo_result in repo_results.items()]
 
     if all(repo_states) and all(parts_states):
@@ -1978,16 +1984,19 @@ def backup_repository_remove(operation_logger, shortname, purge=False):
 
 
 @is_unit_operation()
-def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=0, keep_daily=10, keep_weekly=8, keep_monthly=8):
+def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None, keep_last=None, keep_within=None):
     """
     Remove a backup repository
     """
+
     BackupRepository(shortname).prune(
         prefix=prefix,
         keep_hourly=keep_hourly,
         keep_daily=keep_daily,
         keep_weekly=keep_weekly,
         keep_monthly=keep_monthly,
+        keep_last=keep_last,
+        keep_within=keep_within,
     )
 
 
@@ -2119,7 +2128,7 @@ Group=root
     def run(self):
         self._load_current_values()
         backup_create(
-            name=self.entity,
+            prefix=f"{self.entity}_",
             description=self.description,
             repositories=self.repositories,
             system=self.system,
@@ -2128,7 +2137,7 @@ Group=root
         for repository in self.repositories:
             backup_repository_prune(
                 shortname=repository,
-                prefix=self.entity,
+                prefix=f"{self.entity}_",
                 keep_hourly=self.keep_hourly,
                 keep_daily=self.keep_daily,
                 keep_weekly=self.keep_weekly,

src/repositories/borg.py

@@ -176,45 +176,6 @@ class BorgBackupRepository(LocalBackupRepository):
         response = self._call('info', cmd, json_output=True)
         return response["cache"]["stats"]["unique_size"]
 
-    def prune(self, prefix=None, **kwargs):
-
-        # List archives with creation date
-        archives = {}
-        for archive_name in self.list_archive_name(prefix):
-            archive = BackupArchive(repo=self, name=archive_name)
-            created_at = archive.info()["created_at"]
-            archives[created_at] = archive
-
-        if not archives:
-            return
-
-        # Generate periods in which keep one archive
-        now = datetime.utcnow()
-        now -= timedelta(
-            minutes=now.minute,
-            seconds=now.second,
-            microseconds=now.microsecond
-        )
-        periods = set([])
-
-        for unit, qty in kwargs:
-            if not qty:
-                continue
-            period = timedelta(**{unit: 1})
-            periods += set([(now - period * i, now - period * (i - 1))
-                           for i in range(qty)])
-
-        # Delete unneeded archive
-        for created_at in sorted(archives, reverse=True):
-            created_at = datetime.utcfromtimestamp(created_at)
-            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
-
-            if keep_for:
-                periods -= keep_for
-                continue
-
-            archive.delete()
-
 
 class BorgBackupArchive(BackupArchive):
     """ Backup prepared files with borg """

src/repository.py

@@ -283,7 +283,30 @@ class BackupRepository(ConfigPanel):
 
         return archives
 
-    def prune(self, prefix=None, **kwargs):
+    def prune(self, prefix=None, keep_last=None, keep_within=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None):
+
+        # Default prune options
+        keeps = [value is None for key, value in locals().items() if key.startswith("keep_")]
+        if all(keeps):
+            keep_hourly = 0
+            keep_daily = 10
+            keep_weekly = 8
+            keep_monthly = 8
+            logger.debug(f"Prune and keep one per each {keep_hourly} last hours, {keep_daily} last days, {keep_weekly} last weeks, {keep_monthly} last months")
+
+        keep_last = keep_last if keep_last else 0
+
+        # Convert keep_within as a date
+        units = {
+            "H": "hours",
+            "d": "days",
+            "w": "weeks",
+        }
+        now = datetime.utcnow()
+        if keep_within:
+            keep_within = now - timedelta(**{units[keep_within[-1]]: int(keep_within[:-1])})
+        else:
+            keep_within = now
 
         # List archives with creation date
         archives = {}
@@ -303,24 +326,32 @@ class BackupRepository(ConfigPanel):
             microseconds=now.microsecond
         )
         periods = set([])
-
+        units = {
-        for unit, qty in kwargs:
+            "keep_hourly": {"hours": 1},
+            "keep_daily": {"days": 1},
+            "keep_weekly": {"weeks": 1},
+            "keep_monthly": {"days": 30}
+        }
+        keeps_xly = {key: val for key, val in locals().items()
+                     if key.startswith("keep_") and key.endswith("ly")}
+        for unit, qty in keeps_xly.items():
             if not qty:
                 continue
-            period = timedelta(**{unit: 1})
+            period = timedelta(**units[unit])
-            periods += set([(now - period * i, now - period * (i - 1))
+            periods.update(set([(now - period * i, now - period * (i - 1))
-                           for i in range(qty)])
+                           for i in range(qty)]))
 
         # Delete unneeded archive
         for created_at in sorted(archives, reverse=True):
-            created_at = datetime.utcfromtimestamp(created_at)
+            date_created_at = datetime.utcfromtimestamp(created_at)
-            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
+            keep_for = set(filter(lambda period: period[0] <= date_created_at <= period[1], periods))
-
-            if keep_for:
             periods -= keep_for
+
+            if keep_for or keep_last > 0 or date_created_at >= keep_within:
+                keep_last -= 1
                 continue
 
-            archive.delete()
+            archives[created_at].delete()
 
     # =================================================
     # Repository abstract actions