[wip] Implementation borg method + timer

data/actionsmap/yunohost.yml

@@ -1046,11 +1046,9 @@ backup:
                 -d:
                     full: --description
                     help: Short description of the backup
-                -o:
-                    full: --output-directory
-                    help: Output directory for the backup
-                --methods:
-                    help: List of backup methods to apply (copy or tar by default)
+                -r:
+                    full: --repositories
+                    help: List of repositories where send backup files (local borg repo use by default)
                     nargs: "*"
                 --system:
                     help: List of system parts to backup (or all if none given).
@@ -1085,7 +1083,7 @@ backup:
             api: GET /backups
             arguments:
                 -r:
-                    full: --repos
+                    full: --repositories
                     help: List archives in these repositories
                     nargs: "*"
                 -i:
@@ -1120,7 +1118,7 @@ backup:
             arguments:
                 name:
                     help: Name of the local backup archive
-
+        
         ### backup_delete()
         delete:
             action_help: Delete a backup archive
@@ -1139,7 +1137,7 @@ backup:
                 ### backup_repository_list()
                 list:
                     action_help: List available repositories where put archives
-                    api: GET /backup/repositories
+                    api: GET /backups/repositories
                     arguments:
                         --full:
                             help: Show more details
@@ -1148,8 +1146,14 @@ backup:
                 ### backup_repository_info()
                 info:
                     action_help: Show info about a repository
-                    api: GET /backup/repository
+                    api: GET /backups/repository/<shortname>
                     arguments:
+                        shortname:
+                            help: ID of the repository
+                            extra:
+                                pattern: &pattern_backup_repository_shortname
+                                    - !!str ^[a-zA-Z0-9-_\.]+$
+                                    - "pattern_backup_repository_shortname"
                         -H:
                             full: --human-readable
                             help: Print sizes in human readable format
@@ -1161,14 +1165,12 @@ backup:
                 ### backup_repository_add()
                 add:
                     action_help: Add a backup repository
-                    api: POST /backup/repository/<shortname>
+                    api: POST /backups/repository/<shortname>
                     arguments:
                         shortname:
                             help: ID of the repository
                             extra:
-                                pattern: &pattern_backup_repository_shortname
-                                    - !!str ^[a-zA-Z0-9-_]+$
-                                    - "pattern_backup_repository_shortname"
+                                pattern: *pattern_backup_repository_shortname
                         -n:
                             full: --name
                             help: Short description of the repository
@@ -1182,7 +1184,6 @@ backup:
                         -m:
                             full: --method
                             help: By default 'borg' method is used, could be 'tar' or a custom method
-                            default: borg
                         -q:
                             full: --quota
                             help: Quota to configure with this repository
@@ -1197,14 +1198,13 @@ backup:
                         -d:
                             full: --alert-delay
                             help: Inactivity delay in days after which we sent alerts mails
-                            default: 7
 
                 ### backup_repository_update()
                 update:
                     action_help: Update a backup repository
-                    api: PUT /backup/repository/<name>
+                    api: PUT /backups/repository/<shortname>
                     arguments:
-                        name:
+                        shortname:
                             help: Name of the backup repository to update
                             extra:
                                 pattern: *pattern_backup_repository_shortname
@@ -1224,15 +1224,114 @@ backup:
                 ### backup_repository_remove()
                 remove:
                     action_help: Remove a backup repository
-                    api: DELETE /backup/repository/<name>
+                    api: DELETE /backups/repository/<shortname>
                     arguments:
-                        name:
+                        shortname:
                             help: Name of the backup repository to remove
                             extra:
                                 pattern: *pattern_backup_repository_shortname
                         --purge:
                             help: Remove all archives and data inside repository
-                            action: store_false
+                            action: store_true
+        timer:
+            subcategory_help: Manage backup timer
+            actions:
+                
+                ### backup_timer_list()
+                list:
+                    action_help: List backup timer
+                    api: GET /backup/timer
+                    arguments:
+                        -r:
+                            full: --repositories
+                            help: List archives in these repositories
+                            nargs: "*"
+
+                ### backup_timer_add()
+                add:
+                    action_help: Add a backup timer
+                    api: POST /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
+                            extra:
+                                pattern: &pattern_backup_archive_name
+                                    - !!str ^[\w\-\._]{1,50}(?<!\.)$
+                                    - "pattern_backup_archive_name"
+                        -d:
+                            full: --description
+                            help: Short description of the backup
+                        -r:
+                            full: --repositories
+                            help: List of repositories where send backup files (local borg repo use by default)
+                            nargs: "*"
+                        --system:
+                            help: List of system parts to backup (or all if none given).
+                            nargs: "*"
+                        --apps:
+                            help: List of application names to backup (or all if none given)
+                            nargs: "*"
+                        --schedule:
+                            help: Regular backup frequency (see systemd OnCalendar format)
+                        --alert:
+                            help: Email to alert
+                        --keep-hourly:
+                            default: 2
+                        --keep-daily:
+                            default: 7
+                        --keep-weekly:
+                            default: 8
+                        --keep-monthly:
+                            default: 12
+
+                ### backup_timer_update()
+                update:
+                    action_help: Update a backup timer
+                    api: PUT /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
+                        -d:
+                            full: --description
+                            help: Short description of the backup
+                        -r:
+                            full: --repositories
+                            help: List of repositories where send backup files (local borg repo use by default)
+                            nargs: "*"
+                        --system:
+                            help: List of system parts to backup (or all if none given).
+                            nargs: "*"
+                        --apps:
+                            help: List of application names to backup (or all if none given)
+                            nargs: "*"
+                        --schedule:
+                            help: Regular backup frequency (see systemd OnCalendar format)
+                        --alert:
+                            help: Email to alert
+                        --keep-hourly:
+                            default: 2
+                        --keep-daily:
+                            default: 7
+                        --keep-weekly:
+                            default: 8
+                        --keep-monthly:
+                            default: 12
+
+                ### backup_timer_remove()
+                remove:
+                    action_help: Remove a backup timer
+                    api: DELETE /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
+
+                ### backup_timer_info()
+                info:
+                    action_help: Get info about a backup timer
+                    api: GET /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
 
 #############################
 #         Settings          #

data/other/config_repository.toml

@@ -18,60 +18,60 @@ name.en = ""
         visible = "creation"
         default = "no"
 
-        [main.main.location]
+        [main.main.domain]
         type = "string"
         visible = "creation && is_remote"
         pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        pattern.error = 'location_error' # TODO "Please provide a valid domain"
+        pattern.error = 'domain_error' # TODO "Please provide a valid domain"
         default = ""
         # FIXME: can't be a domain of this instances ?
 
-        [main.main.is_f2f]
+        [main.main.is_shf]
         help = ""
         type = "boolean"
         yes = true
         no = false
         visible = "creation && is_remote"
-        default = "no"
+        default = false
 
         [main.main.public_key]
         type = "alert"
         style = "info"
-        visible = "creation && is_remote && ! is_f2f"
+        visible = "creation && is_remote && ! is_shf"
 
         [main.main.alert]
         help = ''
         type = "tags"
-        visible = "is_remote && is_f2f"
+        visible = "is_remote && is_shf"
         pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        pattern.error = "It seems it's not a valid email"
+        pattern.error = "alert_error"
         default = []
         # "value": alert,
 
         [main.main.alert_delay]
         help = ''
         type = "number"
-        visible = "is_remote && is_f2f"
+        visible = "is_remote && is_shf"
         min = 1
         default = 7
 
         [main.main.quota]
         type = "string"
-        visible = "is_remote && is_f2f"
+        visible = "is_remote && is_shf"
         pattern.regexp = '^\d+[MGT]$'
         pattern.error = '' # TODO ""
         default = ""
 
         [main.main.port]
         type = "number"
-        visible = "is_remote && !is_f2f"
+        visible = "is_remote && !is_shf"
         min = 1
         max = 65535
         default = 22
 
         [main.main.user]
         type = "string"
-        visible = "is_remote && !is_f2f"
+        visible = "is_remote && !is_shf"
         default = ""
 
         [main.main.method]
@@ -84,6 +84,6 @@ name.en = ""
         
         [main.main.path]
         type = "path"
-        visible = "!is_remote or (is_remote and !is_f2f)"
+        visible = "!is_remote or (is_remote and !is_shf)"
         default = "/home/yunohost.backup/archives"
 

locales/en.json

@@ -438,6 +438,7 @@
     "log_app_upgrade": "Upgrade the '{}' app",
     "log_available_on_yunopaste": "This log is now available via {url}",
     "log_backup_create": "Create a backup archive",
+    "log_backup_repository_add": "Add a backup repository",
     "log_backup_restore_app": "Restore '{}' from a backup archive",
     "log_backup_restore_system": "Restore system from a backup archive",
     "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'",
@@ -595,9 +596,9 @@
     "regex_with_only_domain": "You can't use a regex for domain, only for path",
     "repository_config_description": "Long name",
     "repository_config_is_remote": "Remote repository",
-    "repository_config_is_f2f": "It's a YunoHost",
-    "repository_config_is_f2f_help": "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
-    "repository_config_location": "Remote server domain",
+    "repository_config_is_shf": "It's a YunoHost",
+    "repository_config_is_shf_help": "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
+    "repository_config_domain": "Remote server domain",
     "repository_config_public_key": "Public key to give to your BorgBackup  provider : {public_key}",
     "repository_config_alert": "Alert emails",
     "repository_config_alert_help": "Declare emails to which sent inactivity alerts",
@@ -608,6 +609,7 @@
     "repository_config_user": "User",
     "repository_config_method": "Method",
     "repository_config_path": "Archive path",
+    "repository_removed": "Repository '{repository}' removed",
     "restore_already_installed_app": "An app with the ID '{app}' is already installed",
     "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}",
     "restore_backup_too_old": "This backup archive can not be restored because it comes from a too-old YunoHost version.",

src/yunohost/repositories/borg.py

@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2013 Yunohost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+
+class BorgBackupRepository(LocalBackupRepository):
+    need_organized_files = True
+    method_name = "borg"
+
+    # TODO logs
+    def _run_borg_command(self, cmd, stdout=None):
+        """ Call a submethod of borg with the good context
+        """
+        env = dict(os.environ)
+
+        if self.domain:
+            # TODO Use the best/good key
+            private_key = "/root/.ssh/ssh_host_ed25519_key"
+
+            # Don't check ssh fingerprint strictly the first time
+            # TODO improve this by publishing and checking this with DNS
+            strict = 'yes' if self.domain in open('/root/.ssh/known_hosts').read() else 'no'
+            env['BORG_RSH'] = "ssh -i %s -oStrictHostKeyChecking=%s"
+            env['BORG_RSH'] = env['BORG_RSH'] % (private_key, strict)
+
+        # In case, borg need a passphrase to get access to the repo
+        if "passphrase" in self.future_values:
+            env['BORG_PASSPHRASE'] = self.passphrase
+
+        # Authorize to move the repository (borgbase do this)
+        env["BORG_RELOCATED_REPO_ACCESS_IS_OK"] = "yes"
+
+        return subprocess.Popen(cmd, env=env, stdout=stdout)
+
+    def _call(self, action, cmd, json_output=False):
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError(f"backup_borg_{action}_error")
+
+        out, _ = borg.communicate()
+        if json_output:
+            try:
+                return json.loads(out)
+            except json.decoder.JSONDecodeError, TypeError:
+                raise YunohostError(f"backup_borg_{action}_error")
+        return out
+
+    # =================================================
+    # Repository actions
+    # =================================================
+
+    def install(self):
+        # Remote
+        if self.is_remote:
+            if self.is_shf and not self.future_values.get('user'):
+                services = {
+                    'borg': 'borgbackup'
+                }
+
+                response = shf_request(
+                    domain=self.domain,
+                    service=services[self.method],
+                    shf_id=values.pop('shf_id', None),
+                    data = {
+                        'origin': self.domain,
+                        'public_key': self.public_key,
+                        'quota':  self.quota,
+                        'alert':  self.alert,
+                        'alert_delay':  self.alert_delay,
+                        #password: "XXXXXXXX",
+                    }
+                )
+                self.new_values['shf_id'] = response['id']
+                self.new_values['location'] = response['repository']
+            elif not self.is_shf:
+                self.new_values['location'] = self.location
+
+            if not self.future_values.get('user'):
+                raise YunohostError("")
+        # Local
+        else:
+            super().install()
+
+
+        # Initialize borg repo
+        cmd = ["borg", "init", "--encryption", "repokey", self.location]
+
+        if "quota" in self.future_values:
+            cmd += ['--storage-quota', self.quota]
+        self._call('init', cmd)
+
+
+    def update(self):
+        raise NotImplementedError()
+
+    def purge(self):
+        if self.is_shf:
+            response = shf_request(
+                domain=self.domain,
+                service="borgbackup",
+                shf_id=values.pop('shf_id', None),
+                data = {
+                    'origin': self.domain,
+                    #password: "XXXXXXXX",
+                }
+            )
+        else:
+            cmd = ["borg", "delete", self.location]
+            self._call('purge', cmd)
+            if not self.is_remote:
+                super().purge()
+
+    def list_archives_names(self, prefix=None):
+        cmd = ["borg", "list", "--json", self.location]
+        if prefix:
+            cmd += ["-P", prefix]
+        response = self._call('list', cmd, True)
+        return [archive["name"] for archive in response['archives']]
+
+    def compute_space_used(self):
+        if not self.is_remote:
+            return super().purge()
+        else:
+            cmd = ["borg", "info", "--json", self.location]
+            response = self._call('info', cmd)
+            return response["cache"]["stats"]["unique_size"]
+
+    def prune(self, prefix=None, hourly=None, daily=None, weekly=None, yearly=None):
+
+        # List archives with creation date
+        archives = {}
+        for archive_name in self.list_archive_name(prefix):
+            archive = BackupArchive(repo=self, name=archive_name)
+            created_at = archive.info()["created_at"]
+            archives[created_at] = archive
+
+        if not archives:
+            return
+
+        # Generate period in which keep one archive
+        now = datetime.now()
+        periods = []
+        for in range(hourly):
+
+
+
+        for created_at in sorted(archives, reverse=True):
+            created_at = datetime.fromtimestamp(created_at)
+            if hourly > 0 and :
+                hourly-=1
+
+            archive.delete()
+
+
+class BorgBackupArchive(BackupArchive):
+        """ Backup prepared files with borg """
+
+    def backup(self):
+        cmd = ['borg', 'create', self.archive_path, './']
+        self.repo._call('backup', cmd)
+
+    def delete(self):
+        cmd = ['borg', 'delete', '--force', self.archive_path]
+        self.repo._call('delete_archive', cmd)
+
+    def list(self):
+        """ Return a list of archives names
+
+        Exceptions:
+        backup_borg_list_error -- Raised if the borg script failed
+        """
+        cmd = ["borg", "list", "--json-lines", self.archive_path]
+        out = self.repo._call('list_archive', cmd)
+        result = [json.loads(out) for line in out.splitlines()]
+        return result
+
+    def download(self, exclude_paths=[]):
+        super().download()
+        paths = self.select_files()
+        if isinstance(exclude_paths, str):
+            exclude_paths = [exclude_paths]
+        # Here tar archive are not compressed, if we want to compress we
+        # should add --tar-filter=gzip.
+        cmd = ["borg", "export-tar", self.archive_path, "-"] + paths
+        for path in exclude_paths:
+            cmd += ['--exclude', path]
+        reader = self.repo._run_borg_command(cmd, stdout=subprocess.PIPE)
+
+        # We return a raw bottle HTTPresponse (instead of serializable data like
+        # list/dict, ...), which is gonna be picked and used directly by moulinette
+        from bottle import response, HTTPResponse
+        response.content_type = "application/x-tar"
+        return HTTPResponse(reader, 200)
+
+    def extract(self, paths=None, exclude_paths=[]):
+        paths, exclude_paths = super().extract(paths, exclude_paths)
+        cmd = ['borg', 'extract', self.archive_path] + paths
+        for path in exclude_paths:
+            cmd += ['--exclude', path]
+        return self.repo._call('extract_archive', cmd)
+
+    def mount(self, path):
+        # FIXME How to be sure the place where we mount is secure ?
+        cmd = ['borg', 'mount', self.archive_path, path]
+        self.repo._call('mount_archive', cmd)
+
+

src/yunohost/repositories/hook.py

@@ -0,0 +1,128 @@
+
+class HookBackupRepository(BackupRepository):
+    method_name = "hook"
+
+    # =================================================
+    # Repository actions
+    # =================================================
+    def install(self):
+        raise NotImplementedError()
+
+    def update(self):
+        raise NotImplementedError()
+
+    def remove(self, purge=False):
+        if self.__class__ == BackupRepository:
+            raise NotImplementedError() # purge
+
+        rm(self.save_path, force=True)
+        logger.success(m18n.n("repository_removed", repository=self.shortname))
+
+    def list(self):
+        raise NotImplementedError()
+
+    def info(self, space_used=False):
+        result = super().get(mode="export")
+
+        if self.__class__ == BackupRepository and space_used == True:
+            raise NotImplementedError() # purge
+
+        return {self.shortname: result}
+
+    def prune(self):
+        raise NotImplementedError()
+
+
+class HookBackupArchive(BackupArchive):
+    # =================================================
+    # Archive actions
+    # =================================================
+    def backup(self):
+        raise NotImplementedError()
+        """
+        Launch a custom script to backup
+        """
+
+        self._call('backup', self.work_dir, self.name, self.repo.location, self.manager.size,
+                self.manager.description)
+
+    def restore(self):
+        raise NotImplementedError()
+
+    def delete(self):
+        raise NotImplementedError()
+
+    def list(self):
+        raise NotImplementedError()
+        """ Return a list of archives names
+
+        Exceptions:
+        backup_custom_list_error -- Raised if the custom script failed
+        """
+        out = self._call('list', self.repo.location)
+        result = out.strip().splitlines()
+        return result
+
+    def info(self):
+        raise NotImplementedError() #compute_space_used
+        """ Return json string of the info.json file
+
+        Exceptions:
+        backup_custom_info_error -- Raised if the custom script failed
+        """
+        return self._call('info', self.name, self.repo.location)
+
+    def download(self):
+        raise NotImplementedError()
+
+    def mount(self):
+        raise NotImplementedError()
+        """
+        Launch a custom script to mount the custom archive
+        """
+        super().mount()
+        self._call('mount', self.work_dir, self.name, self.repo.location, self.manager.size,
+                self.manager.description)
+
+    def extract(self):
+        raise NotImplementedError()
+
+    def need_organized_files(self):
+        """Call the backup_method hook to know if we need to organize files"""
+        if self._need_mount is not None:
+            return self._need_mount
+
+        try:
+            self._call('nedd_mount')
+        except YunohostError:
+            return False
+        return True
+    def _call(self, *args):
+        """ Call a submethod of backup method hook
+
+        Exceptions:
+        backup_custom_ACTION_error -- Raised if the custom script failed
+        """
+        ret = hook_callback("backup_method", [self.method],
+                            args=args)
+
+        ret_failed = [
+            hook
+            for hook, infos in ret.items()
+            if any(result["state"] == "failed" for result in infos.values())
+        ]
+        if ret_failed:
+            raise YunohostError("backup_custom_" + args[0] + "_error")
+
+        return ret["succeed"][self.method]["stdreturn"]
+
+    def _get_args(self, action):
+        """Return the arguments to give to the custom script"""
+        return [
+            action,
+            self.work_dir,
+            self.name,
+            self.repo,
+            self.manager.size,
+            self.manager.description,
+        ]

src/yunohost/repositories/tar.py

@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2013 Yunohost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+
+class TarBackupRepository(LocalBackupRepository):
+    need_organized_files = False
+    method_name = "tar"
+    def list_archives_names(self):
+        # Get local archives sorted according to last modification time
+        # (we do a realpath() to resolve symlinks)
+        archives = glob(f"{self.location}/*.tar.gz") + glob(f"{self.location}/*.tar")
+        archives = set([os.path.realpath(archive) for archive in archives])
+        archives = sorted(archives, key=lambda x: os.path.getctime(x))
+
+        # Extract only filename without the extension
+        def remove_extension(f):
+            if f.endswith(".tar.gz"):
+                return os.path.basename(f)[: -len(".tar.gz")]
+            else:
+                return os.path.basename(f)[: -len(".tar")]
+
+        return [remove_extension(f) for f in archives]
+
+    def compute_space_used(self):
+        return space_used_in_directory(self.location)
+
+    def prune(self):
+        raise NotImplementedError()
+
+
+class TarBackupArchive:
+    @property
+    def archive_path(self):
+
+        if isinstance(self.manager, BackupManager) and settings_get(
+            "backup.compress_tar_archives"
+        ):
+            return os.path.join(self.repo.location, self.name + ".tar.gz")
+
+        f = os.path.join(self.repo.path, self.name + ".tar")
+        if os.path.exists(f + ".gz"):
+            f += ".gz"
+        return f
+
+    def backup(self):
+        # Open archive file for writing
+        try:
+            tar = tarfile.open(
+                self.archive_path,
+                "w:gz" if self.archive_path.endswith(".gz") else "w",
+            )
+        except Exception:
+            logger.debug(
+                "unable to open '%s' for writing", self.archive_path, exc_info=1
+            )
+            raise YunohostError("backup_archive_open_failed")
+
+        # Add files to the archive
+        try:
+            for path in self.manager.paths_to_backup:
+                # Add the "source" into the archive and transform the path into
+                # "dest"
+                tar.add(path["source"], arcname=path["dest"])
+        except IOError:
+            logger.error(
+                m18n.n(
+                    "backup_archive_writing_error",
+                    source=path["source"],
+                    archive=self._archive_file,
+                    dest=path["dest"],
+                ),
+                exc_info=1,
+            )
+            raise YunohostError("backup_creation_failed")
+        finally:
+            tar.close()
+
+        # Move info file
+        shutil.copy(
+            os.path.join(self.work_dir, "info.json"),
+            os.path.join(ARCHIVES_PATH, self.name + ".info.json"),
+        )
+
+        # If backuped to a non-default location, keep a symlink of the archive
+        # to that location
+        link = os.path.join(self.repo.path, self.name + ".tar")
+        if not os.path.isfile(link):
+            os.symlink(self.archive_path, link)
+
+    def copy(self, file, target):
+        tar = tarfile.open(
+            self._archive_file, "r:gz" if self._archive_file.endswith(".gz") else "r"
+        )
+        file_to_extract = tar.getmember(file)
+        # Remove the path
+        file_to_extract.name = os.path.basename(file_to_extract.name)
+        tar.extract(file_to_extract, path=target)
+        tar.close()
+
+    def delete(self):
+        archive_file = f"{self.repo.location}/{self.name}.tar"
+        info_file = f"{self.repo.location}/{self.name}.info.json"
+        if os.path.exists(archive_file + ".gz"):
+            archive_file += ".gz"
+
+        files_to_delete = [archive_file, info_file]
+
+        # To handle the case where archive_file is in fact a symlink
+        if os.path.islink(archive_file):
+            actual_archive = os.path.realpath(archive_file)
+            files_to_delete.append(actual_archive)
+
+        for backup_file in files_to_delete:
+            if not os.path.exists(backup_file):
+                continue
+            try:
+                os.remove(backup_file)
+            except Exception:
+                logger.debug("unable to delete '%s'", backup_file, exc_info=1)
+                logger.warning(m18n.n("backup_delete_error", path=backup_file))
+
+    def list(self):
+        try:
+            tar = tarfile.open(
+                self.archive_path,
+                "r:gz" if self.archive_path.endswith(".gz") else "r",
+            )
+        except Exception:
+            logger.debug(
+                "cannot open backup archive '%s'", self.archive_path, exc_info=1
+            )
+            raise YunohostError("backup_archive_open_failed")
+
+        try:
+            return tar.getnames()
+        except (IOError, EOFError, tarfile.ReadError) as e:
+            tar.close()
+            raise YunohostError(
+                "backup_archive_corrupted", archive=self.archive_path, error=str(e)
+            )
+
+    def download(self):
+        super().download()
+        # If symlink, retrieve the real path
+        archive_file = self.archive_path
+        if os.path.islink(archive_file):
+            archive_file = os.path.realpath(archive_file)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+                raise YunohostValidationError(
+                    "backup_archive_broken_link", path=archive_file
+                )
+
+        # We return a raw bottle HTTPresponse (instead of serializable data like
+        # list/dict, ...), which is gonna be picked and used directly by moulinette
+        from bottle import static_file
+
+        archive_folder, archive_file_name = archive_file.rsplit("/", 1)
+        return static_file(archive_file_name, archive_folder, download=archive_file_name)
+
+    def extract(self, paths=None, exclude_paths=[]):
+        paths, exclude_paths = super().extract(paths, exclude_paths)
+        # Mount the tarball
+        try:
+            tar = tarfile.open(
+                self.archive_path,
+                "r:gz" if self.archive_path.endswith(".gz") else "r",
+            )
+        except Exception:
+            logger.debug(
+                "cannot open backup archive '%s'", self.archive_path, exc_info=1
+            )
+            raise YunohostError("backup_archive_open_failed")
+
+        subdir_and_files = [
+            tarinfo
+            for tarinfo in tar.getmembers()
+            if (
+               any([tarinfo.name.startswith(path) for path in paths])
+               and all([not tarinfo.name.startswith(path) for path in exclude_paths])
+            )
+        ]
+        tar.extractall(members=subdir_and_files, path=self.work_dir)
+        tar.close()
+
+    def mount(self):
+        raise NotImplementedError()
+
+    def _archive_exists(self):
+        return os.path.lexists(self.archive_path)
+
+    def _assert_archive_exists(self):
+        if not self._archive_exists():
+            raise YunohostError('backup_archive_name_unknown', name=self.name)
+
+        # If symlink, retrieve the real path
+        if os.path.islink(self.archive_path):
+            archive_file = os.path.realpath(self.archive_path)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+                raise YunohostError('backup_archive_broken_link',
+                                    path=archive_file)
+

src/yunohost/repository.py

@@ -32,246 +32,648 @@ import urllib.parse
 
 from moulinette import Moulinette, m18n
 from moulinette.core import MoulinetteError
-from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, read_yaml, write_to_json
+from moulinette.utils.filesystem import read_file, read_yaml, write_to_json, rm, mkdir, chmod, chown
+from moulinette.utils.network import download_text, download_json
 
 
 from yunohost.utils.config import ConfigPanel, Question
 from yunohost.utils.error import YunohostError
-from yunohost.utils.filesystem import binary_to_human
-from yunohost.utils.network import get_ssh_public_key
+from yunohost.utils.filesystem import space_used_in_directory, disk_usage, binary_to_human
+from yunohost.utils.network import get_ssh_public_key, shf_request, SHF_BASE_URL
 from yunohost.log import OperationLogger, is_unit_operation
 
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_DIR = '/etc/yunohost/repositories'
+CACHE_INFO_DIR = "/var/cache/yunohost/{repository}"
 REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
-
-# TODO i18n
-# TODO split COnfigPanel.get to extract "Format result" part and be able to override it
+# TODO split ConfigPanel.get to extract "Format result" part and be able to override it
 # TODO Migration
 # TODO Remove BackupRepository.get_or_create()
 # TODO Backup method
-# TODO auto test F2F by testing .well-known url
 # TODO API params to get description of forms
 # TODO tests
 # TODO detect external hard drive already mounted and suggest it
-# TODO F2F client detection / add / update / delete
+# TODO F2F client delete
 # TODO F2F server
+# TODO i18n pattern error
 
 class BackupRepository(ConfigPanel):
     """
     BackupRepository manage all repository the admin added to the instance
     """
-    @classmethod
-    def get(cls, shortname):
-        # FIXME
-        if name not in cls.repositories:
-            raise YunohostError('backup_repository_doesnt_exists', name=name)
+    entity_type = "backup_repository"
+    save_path_tpl = "/etc/yunohost/backup/repositories/{entity}.yml"
+    save_mode = "full"
+    need_organized_files = True
+    method_name = ""
 
-        return cls.repositories[name]
-
-    def __init__(self, repository):
-        self.repository = repository
-        self.save_mode = "full"
-        super().__init__(
-            config_path=REPOSITORY_CONFIG_PATH,
-            save_path=f"{REPOSITORIES_DIR}/{repository}.yml",
-        )
-
-        #self.method = BackupMethod.get(method, self)
-
-    def set__domain(self, question):
-        # TODO query on domain name .well-known
-        question.value
-
-    def _get_default_values(self):
-        values = super()._get_default_values()
-        values["public_key"] = get_ssh_public_key()
-        return values
-
-    def list(self, with_info=False):
-        return self.method.list(with_info)
-
-    def compute_space_used(self):
-        if self.used is None:
-            try:
-                self.used = self.method.compute_space_used()
-            except (AttributeError, NotImplementedError):
-                self.used = 'unknown'
-        return self.used
-
-    def purge(self):
-        # TODO F2F delete
-        self.method.purge()
-
-    def delete(self, purge=False):
-
-        if purge:
-            self.purge()
-
-        os.system("rm -rf {REPOSITORY_SETTINGS_DIR}/{self.repository}.yml")
-
-
-    def _split_location(self):
+    @staticmethod
+    def split_location(location):
         """
         Split a repository location into protocol, user, domain and path
         """
-        location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+:))?(?P<path>[^\0]+)$'
-        location_match = re.match(location_regex, self.location)
+        if "/" not in location:
+            return { "domain": location }
+
+        location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+):((?P<port>\d+)/)?)?(?P<path>[^:]+)$'
+        location_match = re.match(location_regex, location)
 
         if location_match is None:
             raise YunohostError('backup_repositories_invalid_location',
                                 location=location)
+        return {
+            'protocol': location_match.group('protocol'),
+            'user': location_match.group('user'),
+            'domain': location_match.group('domain'),
+            'port': location_match.group('port'),
+            'path': location_match.group('path')
+        }
 
-        self.protocol = location_match.group('protocol')
-        self.user = location_match.group('user')
-        self.domain = location_match.group('domain')
-        self.path = location_match.group('path')
+    @classmethod
+    def list(cls, space_used=False, full=False):
+        """
+        List available repositories where put archives
+        """
+        repositories = super().list()
 
+        if not full:
+            return repositories
 
-def backup_repository_list(full=False):
-    """
-    List available repositories where put archives
-    """
+        for repo in repositories:
+            try:
+                repositories[repo] = BackupRepository(repo).info(space_used)
+            except Exception as e:
+                logger.error(f"Unable to open repository {repo}")
 
-    try:
-        repositories = [f.rstrip(".yml")
-                        for f in os.listdir(REPOSITORIES_DIR)
-                        if os.path.isfile(f) and f.endswith(".yml")]
-    except FileNotFoundError:
-        repositories = []
-
-    if not full:
         return repositories
 
-    # FIXME: what if one repo.yml is corrupted ?
-    repositories = {repo: BackupRepository(repo).get(mode="export")
-                    for repo in repositories}
 
-    return repositories
+    # =================================================
+    # Config Panel Hooks
+    # =================================================
+
+    def post_ask__domain(self, question):
+        """ Detect if the domain support Self-Hosting Federation protocol
+        """
+        #import requests
+        # FIXME What if remote server is self-signed ?
+        # FIXME What if remote server is unreachable temporarily ?
+        url = SHF_BASE_URL.format(domain=question.value) + "/"
+        try:
+            #r = requests.get(url, timeout=10)
+            download_text(url, timeout=10)
+        except MoulinetteError as e:
+            logger.debug("SHF not running")
+            return { 'is_shf': False }
+        logger.debug("SHF running")
+        return { 'is_shf': True }
 
 
-def backup_repository_info(shortname, human_readable=True, space_used=False):
-    """
-    Show info about a repository
+    # =================================================
+    # Config Panel Override
+    # =================================================
+    def _get_default_values(self):
+        values = super()._get_default_values()
+        # TODO move that in a getter hooks ?
+        values["public_key"] = get_ssh_public_key()
+        return values
 
-    Keyword arguments:
-        name -- Name of the backup repository
-    """
-    Question.operation_logger = operation_logger
-    repository = BackupRepository(shortname)
-    # TODO
-    if space_used:
-        repository.compute_space_used()
+    def _load_current_values(self):
+        super()._load_current_values()
 
-    repository = repository.get(
-        mode="export"
-    )
+        if 'location' in self.values:
+            self.values.update(BackupRepository.split_location(self.values['location']))
+        self.values['is_remote'] = bool(self.values.get('domain'))
 
-    if human_readable:
-        if 'quota' in repository:
-            repository['quota'] = binary_to_human(repository['quota'])
-        if 'used' in repository and isinstance(repository['used'], int):
-            repository['used'] = binary_to_human(repository['used'])
+        if self.values.get('method') == 'tar' and self.values['is_remote']:
+            raise YunohostError("repository_tar_only_local")
 
-    return repository
+        if 'shf_id' in self.values:
+            self.values['is_shf'] = bool(self.values['shf_id'])
+        self._cast_by_method()
 
+    def _parse_pre_answered(self, *args):
+        super()._parse_pre_answered(*args)
+        if 'location'  in self.args:
+            self.args.update(BackupRepository.split_location(self.args['location']))
+        if 'domain' in self.args:
+            self.args['is_remote'] = bool(self.args['domain'])
+            self.args['method'] = "borg"
+        elif self.args.get('method') == 'tar':
+            self.args['is_remote'] = False
+        self._cast_by_method()
 
-@is_unit_operation()
-def backup_repository_add(operation_logger, shortname, name=None, location=None,
-                          method=None, quota=None, passphrase=None,
-                          alert=[], alert_delay=7):
-    """
-    Add a backup repository
-
-    Keyword arguments:
-        location -- Location of the repository (could be a remote location)
-        shortname -- Name of the backup repository
-        name -- An optionnal description
-        quota -- Maximum size quota of the repository
-        encryption -- If available, the kind of encryption to use
-    """
-    # FIXME i18n
-    # Deduce some value from location
-    args = {}
-    args['description'] = name
-    args['creation'] = True
-    if location:
-        args["location"] = location
-        args["is_remote"] = True
-        args["method"] = method if method else "borg"
-        domain_re = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        if re.match(domain_re, location):
-            args["is_f2f"] = True
-        elif location[0] != "/":
-            args["is_f2f"] = False
+    def _apply(self):
+        # Activate / update services
+        if not os.path.exists(self.save_path):
+            self.install()
         else:
-            args["is_remote"] = False
-            args["method"] = method
-    elif method == "tar":
-        args["is_remote"] = False
-    if not location:
-        args["method"] = method
+            self.update()
 
-    args["quota"] = quota
-    args["passphrase"] = passphrase
-    args["alert"]= ",".join(alert) if alert else None
-    args["alert_delay"]= alert_delay
-
-    # TODO validation
-    # TODO activate service in apply (F2F or not)
-    Question.operation_logger = operation_logger
-    repository = BackupRepository(shortname)
-    return repository.set(
-        args=urllib.parse.urlencode(args),
-        operation_logger=operation_logger
-    )
+        # Clean redundant values before to register
+        for prop in ['is_remote', 'domain', 'port', 'user', 'path',
+                     'creation', 'is_shf', 'shortname']:
+            self.values.pop(prop, None)
+            self.new_values.pop(prop, None)
+        super()._apply()
 
 
-@is_unit_operation()
-def backup_repository_update(operation_logger, shortname, name=None,
-                             quota=None, passphrase=None,
-                             alert=[], alert_delay=None):
-    """
-    Update a backup repository
+    # =================================================
+    # BackupMethod encapsulation
+    # =================================================
+    @property
+    def location(self):
+        if not self.future_values:
+            return None
 
-    Keyword arguments:
-        name -- Name of the backup repository
-    """
+        if not self.is_remote:
+            return self.path
+
+        return f"ssh://{self.user}@{self.domain}:{self.port}/{self.path}"
+
+    def _cast_by_method(self):
+        if not self.future_values:
+            return
+
+        if self.__class__ == BackupRepository:
+            if self.method == 'tar':
+                self.__class__ = TarBackupRepository
+            elif self.method == 'borg':
+                self.__class__ = BorgBackupRepository
+            else:
+                self.__class__ = HookBackupRepository
+
+    def _check_is_enough_free_space(self):
+        """
+        Check free space in repository or output directory before to backup
+        """
+        # TODO How to do with distant repo or with deduplicated backup ?
+        backup_size = self.manager.size
+
+        free_space = free_space_in_directory(self.repo)
+
+        if free_space < backup_size:
+            logger.debug(
+                "Not enough space at %s (free: %s / needed: %d)",
+                self.repo,
+                free_space,
+                backup_size,
+            )
+            raise YunohostValidationError("not_enough_disk_space", path=self.repo)
+
+    def remove(self, purge=False):
+        if purge:
+            self._load_current_values()
+            self.purge()
+
+        rm(self.save_path, force=True)
+        logger.success(m18n.n("repository_removed", repository=self.shortname))
+
+    def info(self, space_used=False):
+        result = super().get(mode="export")
+
+        if self.__class__ == BackupRepository and space_used == True:
+            result["space_used"] = self.compute_space_used()
+
+        return {self.shortname: result}
+
+    def list(self, with_info):
+        archives = self.list_archive_name()
+        if with_info:
+            d = OrderedDict()
+            for archive in archives:
+                try:
+                    d[archive] = BackupArchive(repo=self, name=archive).info()
+                except YunohostError as e:
+                    logger.warning(str(e))
+                except Exception:
+                    import traceback
+
+                    logger.warning(
+                        "Could not check infos for archive %s: %s"
+                        % (archive, "\n" + traceback.format_exc())
+                    )
+
+            archives = d
+
+        return archives
+
+    # =================================================
+    # Repository abstract actions
+    # =================================================
+    def install(self):
+        raise NotImplementedError()
+
+    def update(self):
+        raise NotImplementedError()
+
+    def purge(self):
+        raise NotImplementedError()
+
+    def list_archives_names(self):
+        raise NotImplementedError()
+
+    def compute_space_used(self):
+        raise NotImplementedError()
+
+    def prune(self):
+        raise NotImplementedError() # TODO prune
+
+class LocalBackupRepository(BackupRepository):
+    def install(self):
+        self.new_values['location'] = self.location
+        mkdir(self.location, mode=0o0750, parents=True, uid="admin", gid="root", force=True)
+
+    def update(self):
+        self.install()
+
+    def purge(self):
+        rm(self.location, recursive=True, force=True)
+
+
+class BackupArchive:
+    def __init__(self, repo, name=None, manager=None):
+        self.manager = manager
+        self.name = name or manager.name
+        if self.name.endswith(".tar.gz"):
+            self.name = self.name[: -len(".tar.gz")]
+        elif self.name.endswith(".tar"):
+            self.name = self.name[: -len(".tar")]
+        self.repo = repo
+
+        # Cast
+        if self.repo.method_name == 'tar':
+            self.__class__ = TarBackupArchive
+        elif self.repo.method_name == 'borg':
+            self.__class__ = BorgBackupArchive
+        else:
+            self.__class__ = HookBackupArchive
+
+        # Assert archive exists
+        if not isinstance(self.manager, BackupManager) and self.name not in self.repo.list():
+            raise YunohostValidationError("backup_archive_name_unknown", name=name)
+
+    @property
+    def archive_path(self):
+        """Return the archive path"""
+        return self.repo.location + '::' + self.name
+
+    @property
+    def work_dir(self):
+        """
+        Return the working directory
+
+        For a BackupManager, it is the directory where we prepare the files to
+        backup
+
+        For a RestoreManager, it is the directory where we mount the archive
+        before restoring
+        """
+        return self.manager.work_dir
+
+    # This is not a property cause it could be managed in a hook
+    def need_organized_files(self):
+        return self.repo.need_organised_files
+
+    def organize_and_backup(self):
+        """
+        Run the backup on files listed by  the BackupManager instance
+
+        This method shouldn't be overrided, prefer overriding self.backup() and
+        self.clean()
+        """
+        if self.need_organized_files():
+            self._organize_files()
+
+        self.repo.install()
+
+        # Check free space in output
+        self._check_is_enough_free_space()
+        try:
+            self.backup()
+        finally:
+            self.clean()
+
+    def select_files(self):
+        files_in_archive = self.list()
+
+        if "info.json" in files_in_archive:
+            leading_dot = ""
+            yield "info.json"
+        elif "./info.json" in files_in_archive:
+            leading_dot = "./"
+            yield "./info.json"
+        else:
+            logger.debug(
+                "unable to retrieve 'info.json' inside the archive", exc_info=1
+            )
+            raise YunohostError(
+                "backup_archive_cant_retrieve_info_json", archive=self.archive_path
+            )
+        extract_paths = []
+        if f"{leading_dot}backup.csv" in files_in_archive:
+            yield f"{leading_dot}backup.csv"
+        else:
+            # Old backup archive have no backup.csv file
+            pass
+
+        # Extract system parts backup
+        conf_extracted = False
+
+        system_targets = self.manager.targets.list("system", exclude=["Skipped"])
+        apps_targets = self.manager.targets.list("apps", exclude=["Skipped"])
+
+        for system_part in system_targets:
+            if system_part.startswith("conf_"):
+                if conf_extracted:
+                    continue
+                system_part = "conf/"
+                conf_extracted = True
+            else:
+                system_part = system_part.replace("_", "/") + "/"
+            yield leading_dot + system_part
+        yield f"{leading_dot}hook/restore/"
+
+        # Extract apps backup
+        for app in apps_targets:
+            yield f"{leading_dot}apps/{app}"
+
+    def _get_info_string(self):
+        archive_file = "%s/%s.tar" % (self.repo.path, self.name)
+
+        # Check file exist (even if it's a broken symlink)
+        if not os.path.lexists(archive_file):
+            archive_file += ".gz"
+            if not os.path.lexists(archive_file):
+                raise YunohostValidationError("backup_archive_name_unknown", name=name)
+
+        # If symlink, retrieve the real path
+        if os.path.islink(archive_file):
+            archive_file = os.path.realpath(archive_file)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+                raise YunohostValidationError(
+                    "backup_archive_broken_link", path=archive_file
+                )
+        info_file = CACHE_INFO_DIR.format(repository=self.repo.name)
+        mkdir(info_file, mode=0o0700, parents=True, force=True)
+        info_file += f"/{self.name}.info.json"
+
+        if not os.path.exists(info_file):
+            info_dir = tempfile.mkdtemp()
+            try:
+                files_in_archive = self.list()
+                if "info.json" in files_in_archive:
+                    self.extract("info.json")
+                elif "./info.json" in files_in_archive:
+                    self.extract("./info.json")
+                else:
+                    raise YunohostError(
+                        "backup_archive_cant_retrieve_info_json", archive=archive_file
+                    )
+                shutil.move(os.path.join(info_dir, "info.json"), info_file)
+            finally:
+                os.rmdir(info_dir)
+
+        try:
+            return read_file(info_file)
+        except MoulinetteError:
+            logger.debug("unable to load '%s'", info_file, exc_info=1)
+            raise YunohostError('backup_invalid_archive')
+
+    def info(self):
+
+        info_json = self._get_info_string()
+        if not self._info_json:
+            raise YunohostError('backup_info_json_not_implemented')
+        try:
+            info = json.load(info_json)
+        except:
+            logger.debug("unable to load info json", exc_info=1)
+            raise YunohostError('backup_invalid_archive')
+
+        # (legacy) Retrieve backup size
+        # FIXME
+        size = info.get("size", 0)
+        if not size:
+            tar = tarfile.open(
+                archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
+            )
+            size = reduce(
+                lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
+            )
+            tar.close()
+        result = {
+            "path": repo.archive_path,
+            "created_at": datetime.utcfromtimestamp(info["created_at"]),
+            "description": info["description"],
+            "size": size,
+        }
+        if human_readable:
+            result['size'] = binary_to_human(result['size']) + 'B'
+
+        if with_details:
+            system_key = "system"
+            # Historically 'system' was 'hooks'
+            if "hooks" in info.keys():
+                system_key = "hooks"
+
+            if "size_details" in info.keys():
+                for category in ["apps", "system"]:
+                    for name, key_info in info[category].items():
+
+                        if category == "system":
+                            # Stupid legacy fix for weird format between 3.5 and 3.6
+                            if isinstance(key_info, dict):
+                                key_info = key_info.keys()
+                            info[category][name] = key_info = {"paths": key_info}
+                        else:
+                            info[category][name] = key_info
+
+                        if name in info["size_details"][category].keys():
+                            key_info["size"] = info["size_details"][category][name]
+                            if human_readable:
+                                key_info["size"] = binary_to_human(key_info["size"]) + "B"
+                        else:
+                            key_info["size"] = -1
+                            if human_readable:
+                                key_info["size"] = "?"
+
+            result["apps"] = info["apps"]
+            result["system"] = info[system_key]
+            result["from_yunohost_version"] = info.get("from_yunohost_version")
+
+        return info
+
+# TODO move this in BackupManager ?????
+    def clean(self):
+        """
+        Umount sub directories of working dirextories and delete it if temporary
+        """
+        if self.need_organized_files():
+            if not _recursive_umount(self.work_dir):
+                raise YunohostError("backup_cleaning_failed")
+
+        if self.manager.is_tmp_work_dir:
+            filesystem.rm(self.work_dir, True, True)
+    def _organize_files(self):
+        """
+        Mount all csv src in their related path
+
+        The goal is to organize the files app by app and hook by hook, before
+        custom backup method or before the restore operation (in the case of an
+        unorganize archive).
+
+        The usage of binding could be strange for a user because the du -sb
+        command will return that the working directory is big.
+        """
+        paths_needed_to_be_copied = []
+        for path in self.manager.paths_to_backup:
+            src = path["source"]
+
+            if self.manager is RestoreManager:
+                # TODO Support to run this before a restore (and not only before
+                # backup). To do that RestoreManager.unorganized_work_dir should
+                # be implemented
+                src = os.path.join(self.unorganized_work_dir, src)
+
+            dest = os.path.join(self.work_dir, path["dest"])
+            if dest == src:
+                continue
+            dest_dir = os.path.dirname(dest)
+
+            # Be sure the parent dir of destination exists
+            if not os.path.isdir(dest_dir):
+                filesystem.mkdir(dest_dir, parents=True)
+
+            # For directory, attempt to mount bind
+            if os.path.isdir(src):
+                filesystem.mkdir(dest, parents=True, force=True)
+
+                try:
+                    subprocess.check_call(["mount", "--rbind", src, dest])
+                    subprocess.check_call(["mount", "-o", "remount,ro,bind", dest])
+                except Exception:
+                    logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest))
+                    # To check if dest is mounted, use /proc/mounts that
+                    # escape spaces as \040
+                    raw_mounts = read_file("/proc/mounts").strip().split("\n")
+                    mounts = [m.split()[1] for m in raw_mounts]
+                    mounts = [m.replace("\\040", " ") for m in mounts]
+                    if dest in mounts:
+                        subprocess.check_call(["umount", "-R", dest])
+                else:
+                    # Success, go to next file to organize
+                    continue
+
+            # For files, create a hardlink
+            elif os.path.isfile(src) or os.path.islink(src):
+                # Can create a hard link only if files are on the same fs
+                # (i.e. we can't if it's on a different fs)
+                if os.stat(src).st_dev == os.stat(dest_dir).st_dev:
+                    # Don't hardlink /etc/cron.d files to avoid cron bug
+                    # 'NUMBER OF HARD LINKS > 1' see #1043
+                    cron_path = os.path.abspath("/etc/cron") + "."
+                    if not os.path.abspath(src).startswith(cron_path):
+                        try:
+                            os.link(src, dest)
+                        except Exception as e:
+                            # This kind of situation may happen when src and dest are on different
+                            # logical volume ... even though the st_dev check previously match...
+                            # E.g. this happens when running an encrypted hard drive
+                            # where everything is mapped to /dev/mapper/some-stuff
+                            # yet there are different devices behind it or idk ...
+                            logger.warning(
+                                "Could not link %s to %s (%s) ... falling back to regular copy."
+                                % (src, dest, str(e))
+                            )
+                        else:
+                            # Success, go to next file to organize
+                            continue
+
+            # If mountbind or hardlink couldnt be created,
+            # prepare a list of files that need to be copied
+            paths_needed_to_be_copied.append(path)
+
+        if len(paths_needed_to_be_copied) == 0:
+            return
+        # Manage the case where we are not able to use mount bind abilities
+        # It could be just for some small files on different filesystems or due
+        # to mounting error
+
+        # Compute size to copy
+        size = sum(disk_usage(path["source"]) for path in paths_needed_to_be_copied)
+        size /= 1024 * 1024  # Convert bytes to megabytes
+
+        # Ask confirmation for copying
+        if size > MB_ALLOWED_TO_ORGANIZE:
+            try:
+                i = Moulinette.prompt(
+                    m18n.n(
+                        "backup_ask_for_copying_if_needed",
+                        answers="y/N",
+                        size=str(size),
+                    )
+                )
+            except NotImplemented:
+                raise YunohostError("backup_unable_to_organize_files")
+            else:
+                if i != "y" and i != "Y":
+                    raise YunohostError("backup_unable_to_organize_files")
+
+        # Copy unbinded path
+        logger.debug(m18n.n("backup_copying_to_organize_the_archive", size=str(size)))
+        for path in paths_needed_to_be_copied:
+            dest = os.path.join(self.work_dir, path["dest"])
+            if os.path.isdir(path["source"]):
+                shutil.copytree(path["source"], dest, symlinks=True)
+            else:
+                shutil.copy(path["source"], dest)
+
+    # =================================================
+    # Archive abstract actions
+    # =================================================
+    def backup(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+
+    def delete(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+
+    def list(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+
+
+    def download(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+        if Moulinette.interface.type != "api":
+            logger.error(
+                "This option is only meant for the API/webadmin and doesn't make sense for the command line."
+            )
+            return
+
+    def extract(self, paths=None, exclude_paths=[]):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+        if isinstance(exclude_paths, str):
+            paths = [paths]
+        elif paths is None:
+            paths = self.select_files()
+        if isinstance(exclude_paths, str):
+            exclude_paths = [exclude_paths]
+        return paths, exclude_paths
+
+    def mount(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
 
-    args = {}
-    args['creation'] = False
-    if name:
-        args['description'] = name
-    if quota:
-        args["quota"] = quota
-    if passphrase:
-        args["passphrase"] = passphrase
-    if alert is not None:
-        args["alert"]= ",".join(alert) if alert else None
-    if alert_delay:
-        args["alert_delay"]= alert_delay
 
-    # TODO validation
-    # TODO activate service in apply
-    Question.operation_logger = operation_logger
-    repository = BackupRepository(shortname)
-    return repository.set(
-        args=urllib.parse.urlencode(args),
-        operation_logger=operation_logger
-    )
 
 
-@is_unit_operation()
-def backup_repository_remove(operation_logger, shortname, purge=False):
-    """
-    Remove a backup repository
 
-    Keyword arguments:
-        name -- Name of the backup repository to remove
 
-    """
-    BackupRepository(shortname).delete(purge)
-    logger.success(m18n.n('backup_repository_removed', repository=shortname,
-                        path=repository['path']))

src/yunohost/utils/network.py

@@ -29,6 +29,7 @@ from moulinette.utils.process import check_output
 
 logger = logging.getLogger("yunohost.utils.network")
 
+SHF_BASE_URL = "https://{domain}/.well-known/self-hosting-federation/v1"
 
 def get_public_ip(protocol=4):
 
@@ -167,6 +168,9 @@ def _extract_inet(string, skip_netmask=False, skip_loopback=True):
     return result
 
 def get_ssh_public_key():
+    """ Return the prefered public key
+    This is used by the Self-Hosting Federation protocol
+    """
     keys = [
         '/etc/ssh/ssh_host_ed25519_key.pub',
         '/etc/ssh/ssh_host_rsa_key.pub'
@@ -176,3 +180,46 @@ def get_ssh_public_key():
             # We return the key without user and machine name.
             # Providers don't need this info.
             return " ".join(read_file(key).split(" ")[0:2])
+
+def shf_request(domain, service, shf_id=None, data={}):
+    # Get missing info from SHF protocol
+    import requests
+    # We try to get the URL repo through SHFi
+    base_url = SHF_BASE_URL.format(domain=domain)
+    url = f"{base_url}/service/{service}"
+
+    # FIXME add signature mechanism and portection against replay attack
+    # FIXME add password to manage the service ?
+    # FIXME support self-signed destination domain by asking validation to user
+    try:
+        if data is None:
+            r = requests.delete(url, timeout=30)
+        else:
+            if shf_id:
+                r = requests.put(f"{url}/{shf_id}", data=data, timeout=30)
+            else:
+                r = requests.post(url, data=data, timeout=30)
+    # SSL exceptions
+    except requests.exceptions.SSLError:
+        raise MoulinetteError("download_ssl_error", url=url)
+    # Invalid URL
+    except requests.exceptions.ConnectionError:
+        raise MoulinetteError("invalid_url", url=url)
+    # Timeout exceptions
+    except requests.exceptions.Timeout:
+        raise MoulinetteError("download_timeout", url=url)
+    # Unknown stuff
+    except Exception as e:
+        raise MoulinetteError("download_unknown_error", url=url, error=str(e))
+    if r.status_code in [401, 403]:
+        if self.creation:
+            raise YunohostError("repository_shf_creation_{r.status_code}")
+        else:
+            response = r.json()
+            raise YunohostError("repository_shf_update_{r.status_code}", message=response['message'])
+
+    elif r.status_code in [200, 201, 202]:
+        return r.json()
+        # FIXME validate repository and id
+    else:
+        raise YunohostError("repository_shf_invalid")