[wip] Repository add

This commit is contained in:
ljf 2021-10-06 18:25:57 +02:00
parent f0bff8f021
commit 8376cade21
9 changed files with 165 additions and 127 deletions

View file

@ -1166,9 +1166,9 @@ backup:
shortname:
help: ID of the repository
extra:
pattern: &pattern_backup_repository_id
- !!str ^\w+$
- "pattern_backup_repository_id"
pattern: &pattern_backup_repository_shortname
- !!str ^[a-zA-Z0-9-_]+$
- "pattern_backup_repository_shortname"
-n:
full: --name
help: Short description of the repository
@ -1207,7 +1207,7 @@ backup:
name:
help: Name of the backup repository to update
extra:
pattern: *pattern_backup_repository_name
pattern: *pattern_backup_repository_shortname
-d:
full: --description
help: Short description of the repository
@ -1229,7 +1229,7 @@ backup:
name:
help: Name of the backup repository to remove
extra:
pattern: *pattern_backup_repository_name
pattern: *pattern_backup_repository_shortname
--purge:
help: Remove all archives and data inside repository
action: store_false

View file

@ -3,73 +3,79 @@ version = "1.0"
i18n = "repository_config"
[main]
name.en = ""
[]
[main.main]
name.en = ""
# if method == "tar": question["value"] = False
[creation] # TODO "Remote repository"
type = "boolean"
visible = "false"
[name] # TODO "Remote repository"
[main.main.description]
type = "string"
default = ""
[is_remote] # TODO "Remote repository"
[main.main.is_remote]
type = "boolean"
yes = true
no = false
visible = "creation && is_remote"
default = "no"
[is_f2f] # TODO "It's a YunoHost",
help = "" # "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
type = "boolean"
yes = true
no = false
visible = "creation && is_remote"
[public_key] # TODO "Here is the public key to give to your BorgBackup provider : {public_key}"
type = "alert"
style = "info"
visible = "creation && is_remote && ! is_f2f"
[location]
ask = "Remote server domain"
[main.main.location]
ask.en = "{is_remote}"
type = "string"
visible = "creation && is_remote"
pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
pattern.error = '' # TODO "Please provide a valid domain"
default = ""
# FIXME: can't be a domain of this instances ?
[alert] # TODO "Alert emails"
help = '' # TODO Declare emails to which sent inactivity alerts",
[main.main.is_f2f]
ask.en = "{is_remote}"
help = ""
type = "boolean"
yes = true
no = false
visible = "creation && is_remote"
default = "no"
[main.main.public_key]
type = "alert"
style = "info"
visible = "creation && is_remote && ! is_f2f"
[main.main.alert]
help = ''
type = "tags"
visible = "is_remote && is_f2f"
pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
pattern.error = "It seems it's not a valid email"
default = []
# "value": alert,
[alert_delay] # TODO "Alert delay"
help = '' # TODO "After how many inactivity days send email alerts",
[main.main.alert_delay]
help = ''
type = "number"
visible = "is_remote && is_f2f"
min = 1
default = 7
[quota] # TODO "Quota"
[main.main.quota]
type = "string"
visible = "is_remote && is_f2f"
pattern.regexp = '^\d+[MGT]$'
pattern.error = '' # TODO ""
default = ""
[port] # TODO "Port"
[main.main.port]
type = "number"
visible = "is_remote && !is_f2f"
min = 1
max = 65535
default = 22
[user] # TODO User
[main.main.user]
type = "string"
visible = "is_remote && !is_f2f"
default = ""
[method] # TODO "Backup method"
[main.main.method]
type = "select"
# "value": method,
choices.borg = "BorgBackup (recommended)"
@ -77,7 +83,8 @@ name.en = ""
default = "borg"
visible = "!is_remote"
[path] # TODO "Archive path"
[main.main.path]
type = "path"
visible = "!is_remote or (is_remote and !is_f2f)"
default = "/home/yunohost.backup/archives"

View file

@ -593,6 +593,21 @@
"regenconf_would_be_updated": "The configuration would have been updated for category '{category}'",
"regex_incompatible_with_tile": "/!\\ Packagers! Permission '{permission}' has show_tile set to 'true' and you therefore cannot define a regex URL as the main URL",
"regex_with_only_domain": "You can't use a regex for domain, only for path",
"repository_config_description": "Long name",
"repository_config_is_remote": "Remote repository",
"repository_config_is_f2f": "It's a YunoHost",
"repository_config_is_f2f_help": "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
"repository_config_location": "Remote server domain",
"repository_config_public_key": "Public key to give to your BorgBackup provider : {public_key}",
"repository_config_alert": "Alert emails",
"repository_config_alert_help": "Declare emails to which sent inactivity alerts",
"repository_config_alert_delay": "Alert delay",
"repository_config_alert_delay_help": "After how many inactivity days send email alerts",
"repository_config_quota": "Quota",
"repository_config_port": "Port",
"repository_config_user": "User",
"repository_config_method": "Method",
"repository_config_path": "Archive path",
"restore_already_installed_app": "An app with the ID '{app}' is already installed",
"restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}",
"restore_backup_too_old": "This backup archive can not be restored because it comes from a too-old YunoHost version.",

View file

@ -73,7 +73,7 @@ from yunohost.log import OperationLogger, is_unit_operation
from yunohost.repository import BackupRepository
from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.utils.packages import ynh_packages_version
from yunohost.utils.filesystem import free_space_in_directory
from yunohost.utils.filesystem import free_space_in_directory, disk_usage, binary_to_human
from yunohost.settings import settings_get
BACKUP_PATH = "/home/yunohost.backup"
@ -1733,15 +1733,15 @@ class BackupMethod(object):
logger.debug("unable to load info json", exc_info=1)
raise YunohostError('backup_invalid_archive')
# (legacy) Retrieve backup size
# (legacy) Retrieve backup size
# FIXME
size = info.get("size", 0)
if not size:
size = info.get("size", 0)
if not size:
tar = tarfile.open(
archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
)
size = reduce(
lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
)
tar.close()
@ -2167,63 +2167,63 @@ class TarBackupMethod(BackupMethod):
path=archive_file)
def _get_info_string(self):
name = self.name
if name.endswith(".tar.gz"):
name = self.name
if name.endswith(".tar.gz"):
name = name[: -len(".tar.gz")]
elif name.endswith(".tar"):
elif name.endswith(".tar"):
name = name[: -len(".tar")]
archive_file = "%s/%s.tar" % (self.repo.path, name)
archive_file = "%s/%s.tar" % (self.repo.path, name)
# Check file exist (even if it's a broken symlink)
if not os.path.lexists(archive_file):
# Check file exist (even if it's a broken symlink)
if not os.path.lexists(archive_file):
archive_file += ".gz"
if not os.path.lexists(archive_file):
raise YunohostValidationError("backup_archive_name_unknown", name=name)
raise YunohostValidationError("backup_archive_name_unknown", name=name)
# If symlink, retrieve the real path
if os.path.islink(archive_file):
# If symlink, retrieve the real path
if os.path.islink(archive_file):
archive_file = os.path.realpath(archive_file)
# Raise exception if link is broken (e.g. on unmounted external storage)
if not os.path.exists(archive_file):
raise YunohostValidationError(
raise YunohostValidationError(
"backup_archive_broken_link", path=archive_file
)
)
info_file = "%s/%s.info.json" % (self.repo.path, name)
info_file = "%s/%s.info.json" % (self.repo.path, name)
if not os.path.exists(info_file):
if not os.path.exists(info_file):
tar = tarfile.open(
archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
)
info_dir = info_file + ".d"
try:
files_in_archive = tar.getnames()
files_in_archive = tar.getnames()
except (IOError, EOFError, tarfile.ReadError) as e:
raise YunohostError(
raise YunohostError(
"backup_archive_corrupted", archive=archive_file, error=str(e)
)
)
try:
if "info.json" in files_in_archive:
if "info.json" in files_in_archive:
tar.extract("info.json", path=info_dir)
elif "./info.json" in files_in_archive:
elif "./info.json" in files_in_archive:
tar.extract("./info.json", path=info_dir)
else:
else:
raise KeyError
except KeyError:
logger.debug(
logger.debug(
"unable to retrieve '%s' inside the archive", info_file, exc_info=1
)
raise YunohostError(
)
raise YunohostError(
"backup_archive_cant_retrieve_info_json", archive=archive_file
)
)
else:
shutil.move(os.path.join(info_dir, "info.json"), info_file)
shutil.move(os.path.join(info_dir, "info.json"), info_file)
finally:
tar.close()
tar.close()
os.rmdir(info_dir)
try:
@ -2804,24 +2804,28 @@ def backup_delete(name):
import yunohost.repository
def backup_repository_list(full):
def backup_repository_list(full=False):
return yunohost.repository.backup_repository_list(full)
def backup_repository_info(name, human_readable, space_used):
return yunohost.repository.backup_repository_info(name, human_readable, space_used)
def backup_repository_info(shortname, human_readable=True, space_used=False):
return yunohost.repository.backup_repository_info(shortname, human_readable, space_used)
def backup_repository_add(location, name, description, methods, quota, encryption):
return yunohost.repository.backup_repository_add(location, name, description, methods, quota, encryption)
def backup_repository_add(shortname, name=None, location=None,
method=None, quota=None, passphrase=None,
alert=[], alert_delay=7):
return yunohost.repository.backup_repository_add(location, shortname, name, method, quota, passphrase, alert, alert_delay)
def backup_repository_update(name, description, quota, password):
return yunohost.repository.backup_repository_update(name, description, quota, password)
def backup_repository_update(shortname, name=None,
quota=None, passphrase=None,
alert=[], alert_delay=None):
return yunohost.repository.backup_repository_update(shortname, name, quota, passphrase, alert, alert_delay)
def backup_repository_remove(name, purge):
return yunohost.repository.backup_repository_remove(name, purge)
def backup_repository_remove(shortname, purge=False):
return yunohost.repository.backup_repository_remove(shortname, purge)
@ -2882,29 +2886,3 @@ def _recursive_umount(directory):
return everything_went_fine
def disk_usage(path):
# We don't do this in python with os.stat because we don't want
# to follow symlinks
du_output = check_output(["du", "-sb", path], shell=False)
return int(du_output.split()[0])
def binary_to_human(n, customary=False):
"""
Convert bytes or bits into human readable format with binary prefix
Keyword argument:
n -- Number to convert
customary -- Use customary symbol instead of IEC standard
"""
symbols = ("Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
if customary:
symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
prefix = {}
for i, s in enumerate(symbols):
prefix[s] = 1 << (i + 1) * 10
for s in reversed(symbols):
if n >= prefix[s]:
value = float(n) / prefix[s]
return "%.1f%s" % (value, s)
return "%s" % n

View file

@ -30,24 +30,24 @@ import subprocess
import re
import urllib.parse
from moulinette import msignals, m18n
from moulinette import Moulinette, m18n
from moulinette.core import MoulinetteError
from moulinette.utils import filesystem
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, read_yaml, write_to_json
from yunohost.utils.config import ConfigPanel, Question
from yunohost.utils.error import YunohostError
from yunohost.monitor import binary_to_human
from yunohost.utils.filesystem import binary_to_human
from yunohost.utils.network import get_ssh_public_key
from yunohost.log import OperationLogger, is_unit_operation
logger = getActionLogger('yunohost.repository')
REPOSITORIES_PATH = '/etc/yunohost/repositories'
REPOSITORIES_DIR = '/etc/yunohost/repositories'
REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
# TODO
# TODO i18n
# TODO visible in cli
# TODO split COnfigPanel.get to extract "Format result" part and be able to override it
# TODO Migration
# TODO Remove BackupRepository.get_or_create()
@ -76,20 +76,15 @@ class BackupRepository(ConfigPanel):
self.save_mode = "full"
super().__init__(
config_path=REPOSITORY_CONFIG_PATH,
save_path=f"{REPOSITORY_SETTINGS_DIR}/{repository}.yml",
save_path=f"{REPOSITORIES_DIR}/{repository}.yml",
)
#self.method = BackupMethod.get(method, self)
from yunohost.backup import BackupMethod
self.location = location
self._split_location()
self.name = location if name is None else name
if created and self.name in BackupMethod.repositories:
raise YunohostError(
'backup_repository_already_exists', repositories=self.name)
self.method = BackupMethod.get(method, self)
def _get_default_values(self):
values = super()._get_default_values()
values["public_key"] = get_ssh_public_key()
return values
def list(self, with_info=False):
return self.method.list(with_info)
@ -138,7 +133,7 @@ def backup_repository_list(full=False):
try:
repositories = [f.rstrip(".yml")
for f in os.listdir(REPOSITORIES_PATH)
for f in os.listdir(REPOSITORIES_DIR)
if os.path.isfile(f) and f.endswith(".yml")]
except FileNotFoundError:
repositories = []
@ -196,7 +191,7 @@ def backup_repository_add(operation_logger, shortname, name=None, location=None,
# FIXME i18n
# Deduce some value from location
args = {}
args['name'] = name
args['description'] = name
args['creation'] = True
if location:
args["location"] = location
@ -244,7 +239,7 @@ def backup_repository_update(operation_logger, shortname, name=None,
args = {}
args['creation'] = False
if name:
args['name'] = name
args['description'] = name
if quota:
args["quota"] = quota
if passphrase:

View file

@ -86,7 +86,7 @@ class ConfigPanel:
if "ask" in option:
ask = _value_for_locale(option["ask"])
elif "i18n" in self.config:
ask = m18n.n(self.config["i18n"] + "_" + option["id"])
ask = m18n.n(self.config["i18n"] + "_" + option["id"], **self.values)
if mode == "full":
# edit self.config directly
@ -298,7 +298,9 @@ class ConfigPanel:
logger.warning(f"Unknown key '{key}' found in config panel")
# Todo search all i18n keys
out[key] = (
value if key not in ["ask", "help", "name"] else {"en": value}
value
if key not in ["ask", "help", "name"] or isinstance(value, (dict, OrderedDict))
else {"en": value}
)
return out
@ -367,7 +369,7 @@ class ConfigPanel:
if "i18n" in self.config:
for panel, section, option in self._iterate():
if "ask" not in option:
option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"])
option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"], **self.values)
def display_header(message):
"""CLI panel/section header display"""
@ -377,7 +379,8 @@ class ConfigPanel:
for panel, section, obj in self._iterate(["panel", "section"]):
if panel == obj:
name = _value_for_locale(panel["name"])
display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
if name:
display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
continue
name = _value_for_locale(section["name"])
if name:
@ -570,7 +573,7 @@ class Question(object):
def _format_text_for_user_input_in_cli(self):
text_for_user_input_in_cli = _value_for_locale(self.ask)
text_for_user_input_in_cli = _value_for_locale(self.ask).format(**self.context)
if self.choices:

View file

@ -29,3 +29,30 @@ def free_space_in_directory(dirpath):
def space_used_by_directory(dirpath):
stat = os.statvfs(dirpath)
return stat.f_frsize * stat.f_blocks
def disk_usage(path):
# We don't do this in python with os.stat because we don't want
# to follow symlinks
du_output = check_output(["du", "-sb", path], shell=False)
return int(du_output.split()[0])
def binary_to_human(n, customary=False):
"""
Convert bytes or bits into human readable format with binary prefix
Keyword argument:
n -- Number to convert
customary -- Use customary symbol instead of IEC standard
"""
symbols = ("Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
if customary:
symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
prefix = {}
for i, s in enumerate(symbols):
prefix[s] = 1 << (i + 1) * 10
for s in reversed(symbols):
if n >= prefix[s]:
value = float(n) / prefix[s]
return "%.1f%s" % (value, s)
return "%s" % n

View file

@ -18,6 +18,8 @@
along with this program; if not, see http://www.gnu.org/licenses
"""
from collections import OrderedDict
from moulinette import m18n
@ -32,7 +34,7 @@ def _value_for_locale(values):
An utf-8 encoded string
"""
if not isinstance(values, dict):
if not isinstance(values, (dict, OrderedDict)):
return values
for lang in [m18n.locale, m18n.default_locale]:

View file

@ -165,3 +165,14 @@ def _extract_inet(string, skip_netmask=False, skip_loopback=True):
break
return result
def get_ssh_public_key():
keys = [
'/etc/ssh/ssh_host_ed25519_key.pub',
'/etc/ssh/ssh_host_rsa_key.pub'
]
for key in keys:
if os.path.exists(key):
# We return the key without user and machine name.
# Providers don't need this info.
return " ".join(read_file(key).split(" ")[0:2])