YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

[enh] Prune and keep options

Browse source
This commit is contained in:
ljf 2022-10-24 23:18:32 +02:00
parent 5d9b91af96
commit 3a6f1bd612
No known key found for this signature in database
5 changed files with 95 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
locales/en.json
View file

@ -114,6 +114,7 @@
"backup_not_sent": "Backup archive was not saved at all",
"backup_partially_sent": "Backup archive was not sent into all repositories listed",
"backup_nothings_done": "Nothing to save",
"backup_nowhere_to_backup": "Nowhere to backup your file with this archive name",
"backup_output_directory_forbidden": "Pick a different output directory. Backups cannot be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders",
"backup_output_directory_not_empty": "You should pick an empty output directory",
"backup_output_directory_required": "You must provide an output directory for the backup",

34
share/actionsmap.yml
View file

@ -1047,6 +1047,13 @@ backup:
pattern: &pattern_backup_archive_name
- !!str ^[\w\-\._]{1,50}$
- "pattern_backup_archive_name"
-p:
full: --prefix
help: Prefix of the backup archive
extra:
pattern: &pattern_backup_archive_prefix
- !!str ^[\w\-\._]{1,35}$
- "pattern_backup_archive_prefix"
-d:
full: --description
help: Short description of the backup
@ -1277,6 +1284,33 @@ backup:
--prefix:
help: Prefix on which we prune
nargs: "?"
-H:
full: --keep-hourly
help: Number of hourly archives to keep
type: int
-d:
full: --keep-daily
help: Number of daily archives to keep
type: int
-w:
full: --keep-weekly
help: Number of weekly archives to keep
type: int
-m:
full: --keep-monthly
help: Number of monthly archives to keep
type: int
--keep-last:
help: Number of last archives to keep
type: int
--keep-within:
help: Keep all archives within this time interval
extra:
pattern: &pattern_interval
- !!str ^\d+[Hdwmy]$
- "pattern_interval"
timer:
subcategory_help: Manage backup timer
actions:

27
src/backup.py
View file

@ -25,6 +25,7 @@ import csv
import tempfile
import re
import urllib.parse
import datetime
from packaging import version
from moulinette import Moulinette, m18n
@ -260,7 +261,7 @@ class BackupManager:
backup_manager.backup()
"""
def __init__(self, name=None, description="", repositories=[], work_dir=None):
def __init__(self, name=None, prefix="", description="", repositories=[], work_dir=None):
"""
BackupManager constructor
@ -286,7 +287,7 @@ class BackupManager:
self.targets = BackupRestoreTargetsManager()
# Define backup name if needed
self.prefix = prefix
if not name:
name = self._define_backup_name()
self.name = name
@ -334,7 +335,7 @@ class BackupManager:
"""
# FIXME: case where this name already exist
return time.strftime("%Y%m%d-%H%M%S", time.gmtime())
return self.prefix + time.strftime("%Y%m%d-%H%M%S", time.gmtime())
def _init_work_dir(self):
"""Initialize preparation directory
@ -1648,6 +1649,7 @@ class RestoreManager:
def backup_create(
operation_logger,
name=None,
prefix="",
description=None,
repositories=[],
system=[],
@ -1679,13 +1681,15 @@ def backup_create(
# Validate there is no archive with the same name
archives = backup_list(repositories=repositories)
archives_already_exists = []
for repository in archives:
if name and name in archives[repository]:
repositories.pop(repository)
repositories.remove(repository)
archives_already_exists.append(repository)
logger.error(m18n.n("backup_archive_name_exists", repository=repository))
if not repositories:
raise YunohostValidationError("backup_archive_name_exists")
raise YunohostValidationError("backup_nowhere_to_backup")
# If no --system or --apps given, backup everything
@ -1702,7 +1706,8 @@ def backup_create(
repositories = [BackupRepository(repo) for repo in repositories]
# Prepare files to backup
backup_manager = BackupManager(name, description,
backup_manager = BackupManager(name, prefix=prefix,
description=description,
repositories=repositories)
# Add backup targets (system and apps)
@ -1741,6 +1746,7 @@ def backup_create(
)
)
repo_results = backup_manager.backup()
repo_results.update({repo: "Not sent" for repo in archives_already_exists})
repo_states = [repo_result == "Sent" for repository, repo_result in repo_results.items()]
if all(repo_states) and all(parts_states):
@ -1978,16 +1984,19 @@ def backup_repository_remove(operation_logger, shortname, purge=False):
@is_unit_operation()
def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=0, keep_daily=10, keep_weekly=8, keep_monthly=8):
def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None, keep_last=None, keep_within=None):
"""
Remove a backup repository
"""
BackupRepository(shortname).prune(
prefix=prefix,
keep_hourly=keep_hourly,
keep_daily=keep_daily,
keep_weekly=keep_weekly,
keep_monthly=keep_monthly,
keep_last=keep_last,
keep_within=keep_within,
)
@ -2119,7 +2128,7 @@ Group=root
def run(self):
self._load_current_values()
backup_create(
name=self.entity,
prefix=f"{self.entity}_",
description=self.description,
repositories=self.repositories,
system=self.system,
@ -2128,7 +2137,7 @@ Group=root
for repository in self.repositories:
backup_repository_prune(
shortname=repository,
prefix=self.entity,
prefix=f"{self.entity}_",
keep_hourly=self.keep_hourly,
keep_daily=self.keep_daily,
keep_weekly=self.keep_weekly,

39
src/repositories/borg.py
View file

@ -176,45 +176,6 @@ class BorgBackupRepository(LocalBackupRepository):
response = self._call('info', cmd, json_output=True)
return response["cache"]["stats"]["unique_size"]
def prune(self, prefix=None, **kwargs):
# List archives with creation date
archives = {}
for archive_name in self.list_archive_name(prefix):
archive = BackupArchive(repo=self, name=archive_name)
created_at = archive.info()["created_at"]
archives[created_at] = archive
if not archives:
return
# Generate periods in which keep one archive
now = datetime.utcnow()
now -= timedelta(
minutes=now.minute,
seconds=now.second,
microseconds=now.microsecond
)
periods = set([])
for unit, qty in kwargs:
if not qty:
continue
period = timedelta(**{unit: 1})
periods += set([(now - period * i, now - period * (i - 1))
for i in range(qty)])
# Delete unneeded archive
for created_at in sorted(archives, reverse=True):
created_at = datetime.utcfromtimestamp(created_at)
keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
if keep_for:
periods -= keep_for
continue
archive.delete()
class BorgBackupArchive(BackupArchive):
""" Backup prepared files with borg """

53
src/repository.py
View file

@ -283,7 +283,30 @@ class BackupRepository(ConfigPanel):
return archives
def prune(self, prefix=None, **kwargs):
def prune(self, prefix=None, keep_last=None, keep_within=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None):
# Default prune options
keeps = [value is None for key, value in locals().items() if key.startswith("keep_")]
if all(keeps):
keep_hourly = 0
keep_daily = 10
keep_weekly = 8
keep_monthly = 8
logger.debug(f"Prune and keep one per each {keep_hourly} last hours, {keep_daily} last days, {keep_weekly} last weeks, {keep_monthly} last months")
keep_last = keep_last if keep_last else 0
# Convert keep_within as a date
units = {
"H": "hours",
"d": "days",
"w": "weeks",
}
now = datetime.utcnow()
if keep_within:
keep_within = now - timedelta(**{units[keep_within[-1]]: int(keep_within[:-1])})
else:
keep_within = now
# List archives with creation date
archives = {}
@ -303,24 +326,32 @@ class BackupRepository(ConfigPanel):
microseconds=now.microsecond
)
periods = set([])
for unit, qty in kwargs:
units = {
"keep_hourly": {"hours": 1},
"keep_daily": {"days": 1},
"keep_weekly": {"weeks": 1},
"keep_monthly": {"days": 30}
}
keeps_xly = {key: val for key, val in locals().items()
if key.startswith("keep_") and key.endswith("ly")}
for unit, qty in keeps_xly.items():
if not qty:
continue
period = timedelta(**{unit: 1})
periods += set([(now - period * i, now - period * (i - 1))
for i in range(qty)])
period = timedelta(**units[unit])
periods.update(set([(now - period * i, now - period * (i - 1))
for i in range(qty)]))
# Delete unneeded archive
for created_at in sorted(archives, reverse=True):
created_at = datetime.utcfromtimestamp(created_at)
keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
date_created_at = datetime.utcfromtimestamp(created_at)
keep_for = set(filter(lambda period: period[0] <= date_created_at <= period[1], periods))
periods -= keep_for
if keep_for:
periods -= keep_for
if keep_for or keep_last > 0 or date_created_at >= keep_within:
keep_last -= 1
continue
archive.delete()
archives[created_at].delete()
# =================================================
# Repository abstract actions