mirror of
https://github.com/YunoHost/yunohost.git
synced 2024-09-03 20:06:10 +02:00
[wip] Repo pruning
This commit is contained in:
ljf
parent
6f8200d9b1
commit
00f8e95de9
5 changed files with 110 additions and 25 deletions
|
|
@ -1041,7 +1041,7 @@ backup:
|
|||
help: Name of the backup archive
|
||||
extra:
|
||||
pattern: &pattern_backup_archive_name
|
||||
- !!str ^[\w\-\._]{1,50}(?<!\.)$
|
||||
- !!str ^[\w\-\._]{1,50}$
|
||||
- "pattern_backup_archive_name"
|
||||
-d:
|
||||
full: --description
|
||||
|
|
@ -1255,9 +1255,9 @@ backup:
|
|||
name:
|
||||
help: Short prefix of the backup archives
|
||||
extra:
|
||||
pattern: &pattern_backup_archive_name
|
||||
- !!str ^[\w\-\._]{1,50}(?<!\.)$
|
||||
- "pattern_backup_archive_name"
|
||||
pattern: &pattern_backup_timer_name
|
||||
- !!str ^[\w\-\._]{1,50}$
|
||||
- "pattern_backup_timer_name"
|
||||
-d:
|
||||
full: --description
|
||||
help: Short description of the backup
|
||||
|
|
@ -1291,6 +1291,8 @@ backup:
|
|||
arguments:
|
||||
name:
|
||||
help: Short prefix of the backup archives
|
||||
extra:
|
||||
pattern: *pattern_backup_timer_name
|
||||
-d:
|
||||
full: --description
|
||||
help: Short description of the backup
|
||||
|
|
@ -1324,6 +1326,8 @@ backup:
|
|||
arguments:
|
||||
name:
|
||||
help: Short prefix of the backup archives
|
||||
extra:
|
||||
pattern: *pattern_backup_timer_name
|
||||
|
||||
### backup_timer_info()
|
||||
info:
|
||||
|
|
@ -1332,6 +1336,8 @@ backup:
|
|||
arguments:
|
||||
name:
|
||||
help: Short prefix of the backup archives
|
||||
extra:
|
||||
pattern: *pattern_backup_timer_name
|
||||
|
||||
#############################
|
||||
# Settings #
|
||||
|
|
|
|||
0
src/yunohost/repositories/__init__.py
Normal file
0
src/yunohost/repositories/__init__.py
Normal file
|
|
@ -18,6 +18,18 @@
|
|||
along with this program; if not, see http://www.gnu.org/licenses
|
||||
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from moulinette.utils.log import getActionLogger
|
||||
from moulinette import m18n
|
||||
|
||||
from yunohost.utils.error import YunohostError
|
||||
from yunohost.repository import LocalBackupRepository
|
||||
logger = getActionLogger("yunohost.repository")
|
||||
|
||||
|
||||
class BorgBackupRepository(LocalBackupRepository):
|
||||
|
|
@ -59,7 +71,7 @@ class BorgBackupRepository(LocalBackupRepository):
|
|||
if json_output:
|
||||
try:
|
||||
return json.loads(out)
|
||||
except json.decoder.JSONDecodeError, TypeError:
|
||||
except (json.decoder.JSONDecodeError, TypeError):
|
||||
raise YunohostError(f"backup_borg_{action}_error")
|
||||
return out
|
||||
|
||||
|
|
@ -79,12 +91,12 @@ class BorgBackupRepository(LocalBackupRepository):
|
|||
domain=self.domain,
|
||||
service=services[self.method],
|
||||
shf_id=values.pop('shf_id', None),
|
||||
data = {
|
||||
data={
|
||||
'origin': self.domain,
|
||||
'public_key': self.public_key,
|
||||
'quota': self.quota,
|
||||
'alert': self.alert,
|
||||
'alert_delay': self.alert_delay,
|
||||
'quota': self.quota,
|
||||
'alert': self.alert,
|
||||
'alert_delay': self.alert_delay,
|
||||
#password: "XXXXXXXX",
|
||||
}
|
||||
)
|
||||
|
|
@ -99,7 +111,6 @@ class BorgBackupRepository(LocalBackupRepository):
|
|||
else:
|
||||
super().install()
|
||||
|
||||
|
||||
# Initialize borg repo
|
||||
cmd = ["borg", "init", "--encryption", "repokey", self.location]
|
||||
|
||||
|
|
@ -107,7 +118,6 @@ class BorgBackupRepository(LocalBackupRepository):
|
|||
cmd += ['--storage-quota', self.quota]
|
||||
self._call('init', cmd)
|
||||
|
||||
|
||||
def update(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
|
@ -117,7 +127,7 @@ class BorgBackupRepository(LocalBackupRepository):
|
|||
domain=self.domain,
|
||||
service="borgbackup",
|
||||
shf_id=values.pop('shf_id', None),
|
||||
data = {
|
||||
data={
|
||||
'origin': self.domain,
|
||||
#password: "XXXXXXXX",
|
||||
}
|
||||
|
|
@ -143,7 +153,7 @@ class BorgBackupRepository(LocalBackupRepository):
|
|||
response = self._call('info', cmd)
|
||||
return response["cache"]["stats"]["unique_size"]
|
||||
|
||||
def prune(self, prefix=None, hourly=None, daily=None, weekly=None, yearly=None):
|
||||
def prune(self, prefix=None, **kwargs):
|
||||
|
||||
# List archives with creation date
|
||||
archives = {}
|
||||
|
|
@ -155,17 +165,30 @@ class BorgBackupRepository(LocalBackupRepository):
|
|||
if not archives:
|
||||
return
|
||||
|
||||
# Generate period in which keep one archive
|
||||
now = datetime.now()
|
||||
periods = []
|
||||
for in range(hourly):
|
||||
|
||||
# Generate periods in which keep one archive
|
||||
now = datetime.utcnow()
|
||||
now -= timedelta(
|
||||
minutes=now.minute,
|
||||
seconds=now.second,
|
||||
microseconds=now.microsecond
|
||||
)
|
||||
periods = set([])
|
||||
|
||||
for unit, qty in kwargs:
|
||||
if not qty:
|
||||
continue
|
||||
period = timedelta(**{unit: 1})
|
||||
periods += set([(now - period * i, now - period * (i - 1))
|
||||
for i in range(qty)])
|
||||
|
||||
# Delete unneeded archive
|
||||
for created_at in sorted(archives, reverse=True):
|
||||
created_at = datetime.fromtimestamp(created_at)
|
||||
if hourly > 0 and :
|
||||
hourly-=1
|
||||
created_at = datetime.utcfromtimestamp(created_at)
|
||||
keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
|
||||
|
||||
if keep_for:
|
||||
periods -= keep_for
|
||||
continue
|
||||
|
||||
archive.delete()
|
||||
|
||||
|
|
|
|||
|
|
@ -18,11 +18,26 @@
|
|||
along with this program; if not, see http://www.gnu.org/licenses
|
||||
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import tarfile
|
||||
import shutil
|
||||
|
||||
from moulinette.utils.log import getActionLogger
|
||||
from moulinette import m18n
|
||||
|
||||
from yunohost.utils.error import YunohostError, YunohostValidationError
|
||||
from yunohost.repository import LocalBackupRepository
|
||||
from yunohost.backup import BackupManager
|
||||
from yunohost.utils.filesystem import space_used_in_directory
|
||||
from yunohost.settings import settings_get
|
||||
logger = getActionLogger("yunohost.repository")
|
||||
|
||||
|
||||
class TarBackupRepository(LocalBackupRepository):
|
||||
need_organized_files = False
|
||||
method_name = "tar"
|
||||
|
||||
def list_archives_names(self):
|
||||
# Get local archives sorted according to last modification time
|
||||
# (we do a realpath() to resolve symlinks)
|
||||
|
|
@ -96,7 +111,7 @@ class TarBackupArchive:
|
|||
# Move info file
|
||||
shutil.copy(
|
||||
os.path.join(self.work_dir, "info.json"),
|
||||
os.path.join(ARCHIVES_PATH, self.name + ".info.json"),
|
||||
os.path.join(self.repo.location, self.name + ".info.json"),
|
||||
)
|
||||
|
||||
# If backuped to a non-default location, keep a symlink of the archive
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
|
|||
# TODO F2F server
|
||||
# TODO i18n pattern error
|
||||
|
||||
|
||||
class BackupRepository(ConfigPanel):
|
||||
"""
|
||||
BackupRepository manage all repository the admin added to the instance
|
||||
|
|
@ -74,7 +75,7 @@ class BackupRepository(ConfigPanel):
|
|||
Split a repository location into protocol, user, domain and path
|
||||
"""
|
||||
if "/" not in location:
|
||||
return { "domain": location }
|
||||
return {"domain": location}
|
||||
|
||||
location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+):((?P<port>\d+)/)?)?(?P<path>[^:]+)$'
|
||||
location_match = re.match(location_regex, location)
|
||||
|
|
@ -198,10 +199,13 @@ class BackupRepository(ConfigPanel):
|
|||
|
||||
if self.__class__ == BackupRepository:
|
||||
if self.method == 'tar':
|
||||
from yunohost.repositories.tar import TarBackupRepository
|
||||
self.__class__ = TarBackupRepository
|
||||
elif self.method == 'borg':
|
||||
from yunohost.repositories.borg import BorgBackupRepository
|
||||
self.__class__ = BorgBackupRepository
|
||||
else:
|
||||
from yunohost.repositories.hook import HookBackupRepository
|
||||
self.__class__ = HookBackupRepository
|
||||
|
||||
def _check_is_enough_free_space(self):
|
||||
|
|
@ -259,6 +263,45 @@ class BackupRepository(ConfigPanel):
|
|||
|
||||
return archives
|
||||
|
||||
def prune(self, prefix=None, **kwargs):
|
||||
|
||||
# List archives with creation date
|
||||
archives = {}
|
||||
for archive_name in self.list_archive_name(prefix):
|
||||
archive = BackupArchive(repo=self, name=archive_name)
|
||||
created_at = archive.info()["created_at"]
|
||||
archives[created_at] = archive
|
||||
|
||||
if not archives:
|
||||
return
|
||||
|
||||
# Generate periods in which keep one archive
|
||||
now = datetime.utcnow()
|
||||
now -= timedelta(
|
||||
minutes=now.minute,
|
||||
seconds=now.second,
|
||||
microseconds=now.microsecond
|
||||
)
|
||||
periods = set([])
|
||||
|
||||
for unit, qty in kwargs:
|
||||
if not qty:
|
||||
continue
|
||||
period = timedelta(**{unit: 1})
|
||||
periods += set([(now - period * i, now - period * (i - 1))
|
||||
for i in range(qty)])
|
||||
|
||||
# Delete unneeded archive
|
||||
for created_at in sorted(archives, reverse=True):
|
||||
created_at = datetime.utcfromtimestamp(created_at)
|
||||
keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
|
||||
|
||||
if keep_for:
|
||||
periods -= keep_for
|
||||
continue
|
||||
|
||||
archive.delete()
|
||||
|
||||
# =================================================
|
||||
# Repository abstract actions
|
||||
# =================================================
|
||||
|
|
@ -277,8 +320,6 @@ class BackupRepository(ConfigPanel):
|
|||
def compute_space_used(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def prune(self):
|
||||
raise NotImplementedError() # TODO prune
|
||||
|
||||
class LocalBackupRepository(BackupRepository):
|
||||
def install(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue