From e8c0be1e56c82d8c101cd5fe834b1761b254e5aa Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 10 Oct 2022 02:13:15 +0200 Subject: [PATCH] [fix] Syntax and import error --- share/actionsmap.yml | 3 + ...ory.toml => config_backup_repository.toml} | 0 share/config_backup_timer.toml | 89 +++++++++++++++++++ src/backup.py | 82 +++++++---------- src/repository.py | 44 +++++---- 5 files changed, 143 insertions(+), 75 deletions(-) rename share/{config_repository.toml => config_backup_repository.toml} (100%) create mode 100644 share/config_backup_timer.toml diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 908c32f22..d4fadf1d5 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -1021,6 +1021,9 @@ backup: arguments: name: help: Name of the local backup archive + -r: + full: --repository + help: The archive repository (local borg repo use by default) --system: help: List of system parts to restore (or all if none is given) nargs: "*" diff --git a/share/config_repository.toml b/share/config_backup_repository.toml similarity index 100% rename from share/config_repository.toml rename to share/config_backup_repository.toml diff --git a/share/config_backup_timer.toml b/share/config_backup_timer.toml new file mode 100644 index 000000000..cc0c5290f --- /dev/null +++ b/share/config_backup_timer.toml @@ -0,0 +1,89 @@ + +version = "1.0" +i18n = "repository_config" +[main] +name.en = "" + [main.main] + name.en = "" + optional = false + # if method == "tar": question["value"] = False + [main.main.description] + type = "string" + default = "" + + [main.main.is_remote] + type = "boolean" + yes = true + no = false + visible = "creation" + default = "no" + + [main.main.domain] + type = "string" + visible = "creation && is_remote" + pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$' + pattern.error = 'domain_error' # TODO "Please provide a valid domain" + default = "" + # FIXME: can't be a domain of this instances ? + + [main.main.is_shf] + help = "" + type = "boolean" + yes = true + no = false + visible = "creation && is_remote" + default = false + + [main.main.public_key] + type = "alert" + style = "info" + visible = "creation && is_remote && ! is_shf" + + [main.main.alert] + help = '' + type = "tags" + visible = "is_remote && is_shf" + pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$' + pattern.error = "alert_error" + default = [] + # "value": alert, + + [main.main.alert_delay] + help = '' + type = "number" + visible = "is_remote && is_shf" + min = 1 + default = 7 + + [main.main.quota] + type = "string" + visible = "is_remote && is_shf" + pattern.regexp = '^\d+[MGT]$' + pattern.error = '' # TODO "" + default = "" + + [main.main.port] + type = "number" + visible = "is_remote && !is_shf" + min = 1 + max = 65535 + default = 22 + + [main.main.user] + type = "string" + visible = "is_remote && !is_shf" + default = "" + + [main.main.method] + type = "select" + # "value": method, + choices.borg = "BorgBackup (recommended)" + choices.tar = "Legacy tar archive mechanism" + default = "borg" + visible = "!is_remote" + + [main.main.path] + type = "path" + visible = "!is_remote or (is_remote and !is_shf)" + default = "/home/yunohost.backup/archives" + diff --git a/src/backup.py b/src/backup.py index f8fa6474a..42fdc2416 100644 --- a/src/backup.py +++ b/src/backup.py @@ -32,13 +32,12 @@ import csv import tempfile import re import urllib.parse -from datetime import datetime from packaging import version from moulinette import Moulinette, m18n from moulinette.utils import filesystem from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import mkdir, write_to_yaml, read_yaml, write_to_file +from moulinette.utils.filesystem import mkdir, write_to_yaml, read_yaml, write_to_file, rm from moulinette.utils.process import check_output import yunohost.domain @@ -64,7 +63,7 @@ from yunohost.tools import ( from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger, is_unit_operation from yunohost.repository import BackupRepository, BackupArchive -from yunohost.config import ConfigPanel +from yunohost.utils.config import ConfigPanel from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.packages import ynh_packages_version from yunohost.utils.filesystem import free_space_in_directory, disk_usage, binary_to_human @@ -74,7 +73,6 @@ ARCHIVES_PATH = f"{BACKUP_PATH}/archives" APP_MARGIN_SPACE_SIZE = 100 # In MB CONF_MARGIN_SPACE_SIZE = 10 # IN MB POSTINSTALL_ESTIMATE_SPACE_SIZE = 5 # In MB -MB_ALLOWED_TO_ORGANIZE = 10 logger = getActionLogger("yunohost.backup") @@ -367,6 +365,14 @@ class BackupManager: filesystem.rm(self.work_dir, recursive=True, force=True) filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin") + def clean_work_dir(self, umount=True): + + if umount and not _recursive_umount(self.work_dir): + raise YunohostError("backup_cleaning_failed") + + if self.is_tmp_work_dir: + rm(self.work_dir, True, True) + # # Backup target management # # @@ -875,18 +881,19 @@ class RestoreManager: return restore_manager.result """ - def __init__(self, name, method="tar"): + def __init__(self, archive): """ RestoreManager constructor Args: - name -- (string) Archive name - method -- (string) Method name to use to mount the archive + archive -- (BackupArchive) The archive to restore """ # Retrieve and open the archive # FIXME this way to get the info is not compatible with copy or custom + self.archive = archive + # backup methods - self.info = backup_info(name, with_details=True) + self.info = archive.info() # FIXME with_details=True from_version = self.info.get("from_yunohost_version", "") # Remove any '~foobar' in the version ... c.f ~alpha, ~beta version during @@ -896,9 +903,6 @@ class RestoreManager: if not from_version or version.parse(from_version) < version.parse("4.2.0"): raise YunohostValidationError("restore_backup_too_old") - self.archive_path = self.info["path"] - self.name = name - self.method = BackupMethod.create(method, self) self.targets = BackupRestoreTargetsManager() # @@ -913,32 +917,6 @@ class RestoreManager: return len(successful_apps) != 0 or len(successful_system) != 0 - def _read_info_files(self): - """ - Read the info file from inside an archive - """ - # Retrieve backup info - info_file = os.path.join(self.work_dir, "info.json") - try: - with open(info_file, "r") as f: - self.info = json.load(f) - - # Historically, "system" was "hooks" - - if "system" not in self.info.keys(): - self.info["system"] = self.info["hooks"] - except IOError: - logger.debug("unable to load '%s'", info_file, exc_info=1) - raise YunohostError( - "backup_archive_cant_retrieve_info_json", archive=self.archive_path - ) - else: - logger.debug( - "restoring from backup '%s' created on %s", - self.name, - datetime.utcfromtimestamp(self.info["created_at"]), - ) - def _postinstall_if_needed(self): """ Post install yunohost if needed @@ -1041,10 +1019,8 @@ class RestoreManager: for hook_path in hook_paths: logger.debug( - "Adding restoration script '%s' to the system " - "from the backup archive '%s'", - hook_path, - self.archive_path, + f"Adding restoration script '{hook_path}' to the system " + f"from the backup archive '{self.archive.archive_path}'" ) self.method.copy(hook_path, custom_restore_hook_folder) @@ -1096,7 +1072,7 @@ class RestoreManager: this archive """ - self.work_dir = os.path.join(BACKUP_PATH, "tmp", self.name) + self.work_dir = os.path.join(BACKUP_PATH, "tmp", self.archive.name) if os.path.ismount(self.work_dir): logger.debug("An already mounting point '%s' already exists", self.work_dir) @@ -1120,9 +1096,7 @@ class RestoreManager: filesystem.mkdir(self.work_dir, parents=True) - self.method.extract() - - self._read_info_files() + self.archive.extract() # # Space computation / checks # @@ -1736,7 +1710,7 @@ def backup_create( } -def backup_restore(name, system=[], apps=[], force=False): +def backup_restore(name, repository, system=[], apps=[], force=False): """ Restore from a local backup archive @@ -1757,6 +1731,9 @@ def backup_restore(name, system=[], apps=[], force=False): system = [] apps = [] + if not repository: + repository = "local-borg" + # # Initialize # # @@ -1766,7 +1743,10 @@ def backup_restore(name, system=[], apps=[], force=False): elif name.endswith(".tar"): name = name[: -len(".tar")] - restore_manager = RestoreManager(name) + repo = BackupRepository(repository) + archive = BackupArchive(name, repo) + + restore_manager = RestoreManager(archive) restore_manager.set_system_targets(system) restore_manager.set_apps_targets(apps) @@ -1827,7 +1807,7 @@ def backup_list(repositories=[], with_info=False, human_readable=False): """ return { - name: BackupRepository(name).list(with_info) + name: BackupRepository(name).list_archives(with_info) for name in repositories or BackupRepository.list(full=False) } @@ -1862,7 +1842,7 @@ def backup_info(name, repository=None, with_details=False, human_readable=False) repo = BackupRepository(repository) archive = BackupArchive(name, repo) - return archive.info() + return archive.info(with_details=with_details, human_readable=human_readable) def backup_delete(name, repository): @@ -1969,7 +1949,7 @@ class BackupTimer(ConfigPanel): Description=Run backup {self.entity} regularly [Timer] -OnCalendar={values['schedule']} +OnCalendar={self.values['schedule']} [Install] WantedBy=timers.target @@ -1982,7 +1962,7 @@ After=network.target [Service] Type=oneshot -ExecStart=/usr/bin/yunohost backup create -n '{self.entity}' -r '{repo}' --system --apps ; /usr/bin/yunohost backup prune -n '{self.entity}' +ExecStart=/usr/bin/yunohost backup create -n '{self.entity}' -r '{self.repositories}' --system --apps ; /usr/bin/yunohost backup prune -n '{self.entity}' User=root Group=root """) diff --git a/src/repository.py b/src/repository.py index a79bffab5..5af841591 100644 --- a/src/repository.py +++ b/src/repository.py @@ -30,6 +30,7 @@ import shutil import subprocess import tarfile import tempfile +from functools import reduce from moulinette import Moulinette, m18n from moulinette.core import MoulinetteError @@ -38,7 +39,7 @@ from moulinette.utils.filesystem import read_file, rm, mkdir from moulinette.utils.network import download_text from datetime import timedelta, datetime - +import yunohost.repositories from yunohost.utils.config import ConfigPanel from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import disk_usage, binary_to_human, free_space_in_directory @@ -48,6 +49,7 @@ logger = getActionLogger('yunohost.repository') REPOSITORIES_DIR = '/etc/yunohost/repositories' CACHE_INFO_DIR = "/var/cache/yunohost/{repository}" REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml" +MB_ALLOWED_TO_ORGANIZE = 10 # TODO split ConfigPanel.get to extract "Format result" part and be able to override it # TODO Migration # TODO Remove BackupRepository.get_or_create() @@ -240,7 +242,7 @@ class BackupRepository(ConfigPanel): return {self.shortname: result} - def list(self, with_info): + def list_archives(self, with_info): archives = self.list_archive_name() if with_info: d = {} @@ -343,14 +345,14 @@ class BackupArchive: # Cast if self.repo.method_name == 'tar': - self.__class__ = TarBackupArchive + self.__class__ = yunohost.repositories.tar.TarBackupArchive elif self.repo.method_name == 'borg': - self.__class__ = BorgBackupArchive + self.__class__ = yunohost.repositories.borg.BorgBackupArchive else: - self.__class__ = HookBackupArchive + self.__class__ = yunohost.repositories.hook.HookBackupArchive # Assert archive exists - if not isinstance(self.manager, BackupManager) and self.name not in self.repo.list(): + if self.manager.__class__.__name__ != "BackupManager" and self.name not in self.repo.list_archives(): raise YunohostValidationError("backup_archive_name_unknown", name=name) @property @@ -439,17 +441,17 @@ class BackupArchive: yield f"{leading_dot}apps/{app}" def _get_info_string(self): - archive_file = "%s/%s.tar" % (self.repo.path, self.name) + self.archive_file = "%s/%s.tar" % (self.repo.path, self.name) # Check file exist (even if it's a broken symlink) - if not os.path.lexists(archive_file): - archive_file += ".gz" - if not os.path.lexists(archive_file): + if not os.path.lexists(self.archive_file): + self.archive_file += ".gz" + if not os.path.lexists(self.archive_file): raise YunohostValidationError("backup_archive_name_unknown", name=self.name) # If symlink, retrieve the real path - if os.path.islink(archive_file): - archive_file = os.path.realpath(archive_file) + if os.path.islink(self.archive_file): + archive_file = os.path.realpath(self.archive_file) # Raise exception if link is broken (e.g. on unmounted external storage) if not os.path.exists(archive_file): @@ -470,7 +472,7 @@ class BackupArchive: self.extract("./info.json") else: raise YunohostError( - "backup_archive_cant_retrieve_info_json", archive=archive_file + "backup_archive_cant_retrieve_info_json", archive=self.archive_file ) shutil.move(os.path.join(info_dir, "info.json"), info_file) finally: @@ -482,7 +484,7 @@ class BackupArchive: logger.debug("unable to load '%s'", info_file, exc_info=1) raise YunohostError('backup_invalid_archive') - def info(self): + def info(self, with_details, human_readable): info_json = self._get_info_string() if not self._info_json: @@ -498,14 +500,14 @@ class BackupArchive: size = info.get("size", 0) if not size: tar = tarfile.open( - archive_file, "r:gz" if archive_file.endswith(".gz") else "r" + self.archive_file, "r:gz" if self.archive_file.endswith(".gz") else "r" ) size = reduce( lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers() ) tar.close() result = { - "path": repo.archive_path, + "path": self.repo.archive_path, "created_at": datetime.utcfromtimestamp(info["created_at"]), "description": info["description"], "size": size, @@ -546,17 +548,11 @@ class BackupArchive: return info -# TODO move this in BackupManager ????? def clean(self): """ Umount sub directories of working dirextories and delete it if temporary """ - if self.need_organized_files(): - if not _recursive_umount(self.work_dir): - raise YunohostError("backup_cleaning_failed") - - if self.manager.is_tmp_work_dir: - rm(self.work_dir, True, True) + self.manager.clean_work_dir(self.need_organized_files()) def _organize_files(self): """ @@ -573,7 +569,7 @@ class BackupArchive: for path in self.manager.paths_to_backup: src = path["source"] - if self.manager is RestoreManager: + if self.manager.__class__.__name__ != "RestoreManager": # TODO Support to run this before a restore (and not only before # backup). To do that RestoreManager.unorganized_work_dir should # be implemented