diff --git a/locales/en.json b/locales/en.json index 8a8ef457c..3121c3e51 100644 --- a/locales/en.json +++ b/locales/en.json @@ -114,6 +114,7 @@ "backup_not_sent": "Backup archive was not saved at all", "backup_partially_sent": "Backup archive was not sent into all repositories listed", "backup_nothings_done": "Nothing to save", + "backup_nowhere_to_backup": "Nowhere to backup your file with this archive name", "backup_output_directory_forbidden": "Pick a different output directory. Backups cannot be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders", "backup_output_directory_not_empty": "You should pick an empty output directory", "backup_output_directory_required": "You must provide an output directory for the backup", diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 062ec782c..9fbfab049 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -1047,6 +1047,13 @@ backup: pattern: &pattern_backup_archive_name - !!str ^[\w\-\._]{1,50}$ - "pattern_backup_archive_name" + -p: + full: --prefix + help: Prefix of the backup archive + extra: + pattern: &pattern_backup_archive_prefix + - !!str ^[\w\-\._]{1,35}$ + - "pattern_backup_archive_prefix" -d: full: --description help: Short description of the backup @@ -1277,6 +1284,33 @@ backup: --prefix: help: Prefix on which we prune nargs: "?" + -H: + full: --keep-hourly + help: Number of hourly archives to keep + type: int + -d: + full: --keep-daily + help: Number of daily archives to keep + type: int + -w: + full: --keep-weekly + help: Number of weekly archives to keep + type: int + -m: + full: --keep-monthly + help: Number of monthly archives to keep + type: int + --keep-last: + help: Number of last archives to keep + type: int + --keep-within: + help: Keep all archives within this time interval + extra: + pattern: &pattern_interval + - !!str ^\d+[Hdwmy]$ + - "pattern_interval" + + timer: subcategory_help: Manage backup timer actions: diff --git a/src/backup.py b/src/backup.py index e394c5a42..bf3abaeca 100644 --- a/src/backup.py +++ b/src/backup.py @@ -25,6 +25,7 @@ import csv import tempfile import re import urllib.parse +import datetime from packaging import version from moulinette import Moulinette, m18n @@ -260,7 +261,7 @@ class BackupManager: backup_manager.backup() """ - def __init__(self, name=None, description="", repositories=[], work_dir=None): + def __init__(self, name=None, prefix="", description="", repositories=[], work_dir=None): """ BackupManager constructor @@ -286,7 +287,7 @@ class BackupManager: self.targets = BackupRestoreTargetsManager() # Define backup name if needed - + self.prefix = prefix if not name: name = self._define_backup_name() self.name = name @@ -334,7 +335,7 @@ class BackupManager: """ # FIXME: case where this name already exist - return time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + return self.prefix + time.strftime("%Y%m%d-%H%M%S", time.gmtime()) def _init_work_dir(self): """Initialize preparation directory @@ -1648,6 +1649,7 @@ class RestoreManager: def backup_create( operation_logger, name=None, + prefix="", description=None, repositories=[], system=[], @@ -1679,13 +1681,15 @@ def backup_create( # Validate there is no archive with the same name archives = backup_list(repositories=repositories) + archives_already_exists = [] for repository in archives: if name and name in archives[repository]: - repositories.pop(repository) + repositories.remove(repository) + archives_already_exists.append(repository) logger.error(m18n.n("backup_archive_name_exists", repository=repository)) if not repositories: - raise YunohostValidationError("backup_archive_name_exists") + raise YunohostValidationError("backup_nowhere_to_backup") # If no --system or --apps given, backup everything @@ -1702,7 +1706,8 @@ def backup_create( repositories = [BackupRepository(repo) for repo in repositories] # Prepare files to backup - backup_manager = BackupManager(name, description, + backup_manager = BackupManager(name, prefix=prefix, + description=description, repositories=repositories) # Add backup targets (system and apps) @@ -1741,6 +1746,7 @@ def backup_create( ) ) repo_results = backup_manager.backup() + repo_results.update({repo: "Not sent" for repo in archives_already_exists}) repo_states = [repo_result == "Sent" for repository, repo_result in repo_results.items()] if all(repo_states) and all(parts_states): @@ -1978,16 +1984,19 @@ def backup_repository_remove(operation_logger, shortname, purge=False): @is_unit_operation() -def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=0, keep_daily=10, keep_weekly=8, keep_monthly=8): +def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None, keep_last=None, keep_within=None): """ Remove a backup repository """ + BackupRepository(shortname).prune( prefix=prefix, keep_hourly=keep_hourly, keep_daily=keep_daily, keep_weekly=keep_weekly, keep_monthly=keep_monthly, + keep_last=keep_last, + keep_within=keep_within, ) @@ -2119,7 +2128,7 @@ Group=root def run(self): self._load_current_values() backup_create( - name=self.entity, + prefix=f"{self.entity}_", description=self.description, repositories=self.repositories, system=self.system, @@ -2128,7 +2137,7 @@ Group=root for repository in self.repositories: backup_repository_prune( shortname=repository, - prefix=self.entity, + prefix=f"{self.entity}_", keep_hourly=self.keep_hourly, keep_daily=self.keep_daily, keep_weekly=self.keep_weekly, diff --git a/src/repositories/borg.py b/src/repositories/borg.py index 295eeeb16..e18fdb5a4 100644 --- a/src/repositories/borg.py +++ b/src/repositories/borg.py @@ -176,45 +176,6 @@ class BorgBackupRepository(LocalBackupRepository): response = self._call('info', cmd, json_output=True) return response["cache"]["stats"]["unique_size"] - def prune(self, prefix=None, **kwargs): - - # List archives with creation date - archives = {} - for archive_name in self.list_archive_name(prefix): - archive = BackupArchive(repo=self, name=archive_name) - created_at = archive.info()["created_at"] - archives[created_at] = archive - - if not archives: - return - - # Generate periods in which keep one archive - now = datetime.utcnow() - now -= timedelta( - minutes=now.minute, - seconds=now.second, - microseconds=now.microsecond - ) - periods = set([]) - - for unit, qty in kwargs: - if not qty: - continue - period = timedelta(**{unit: 1}) - periods += set([(now - period * i, now - period * (i - 1)) - for i in range(qty)]) - - # Delete unneeded archive - for created_at in sorted(archives, reverse=True): - created_at = datetime.utcfromtimestamp(created_at) - keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods)) - - if keep_for: - periods -= keep_for - continue - - archive.delete() - class BorgBackupArchive(BackupArchive): """ Backup prepared files with borg """ diff --git a/src/repository.py b/src/repository.py index 653043dd6..5164ff779 100644 --- a/src/repository.py +++ b/src/repository.py @@ -283,7 +283,30 @@ class BackupRepository(ConfigPanel): return archives - def prune(self, prefix=None, **kwargs): + def prune(self, prefix=None, keep_last=None, keep_within=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None): + + # Default prune options + keeps = [value is None for key, value in locals().items() if key.startswith("keep_")] + if all(keeps): + keep_hourly = 0 + keep_daily = 10 + keep_weekly = 8 + keep_monthly = 8 + logger.debug(f"Prune and keep one per each {keep_hourly} last hours, {keep_daily} last days, {keep_weekly} last weeks, {keep_monthly} last months") + + keep_last = keep_last if keep_last else 0 + + # Convert keep_within as a date + units = { + "H": "hours", + "d": "days", + "w": "weeks", + } + now = datetime.utcnow() + if keep_within: + keep_within = now - timedelta(**{units[keep_within[-1]]: int(keep_within[:-1])}) + else: + keep_within = now # List archives with creation date archives = {} @@ -303,24 +326,32 @@ class BackupRepository(ConfigPanel): microseconds=now.microsecond ) periods = set([]) - - for unit, qty in kwargs: + units = { + "keep_hourly": {"hours": 1}, + "keep_daily": {"days": 1}, + "keep_weekly": {"weeks": 1}, + "keep_monthly": {"days": 30} + } + keeps_xly = {key: val for key, val in locals().items() + if key.startswith("keep_") and key.endswith("ly")} + for unit, qty in keeps_xly.items(): if not qty: continue - period = timedelta(**{unit: 1}) - periods += set([(now - period * i, now - period * (i - 1)) - for i in range(qty)]) + period = timedelta(**units[unit]) + periods.update(set([(now - period * i, now - period * (i - 1)) + for i in range(qty)])) # Delete unneeded archive for created_at in sorted(archives, reverse=True): - created_at = datetime.utcfromtimestamp(created_at) - keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods)) + date_created_at = datetime.utcfromtimestamp(created_at) + keep_for = set(filter(lambda period: period[0] <= date_created_at <= period[1], periods)) + periods -= keep_for - if keep_for: - periods -= keep_for + if keep_for or keep_last > 0 or date_created_at >= keep_within: + keep_last -= 1 continue - archive.delete() + archives[created_at].delete() # ================================================= # Repository abstract actions