From 04cc1184e47fb2169924ee83218a26d689d2522d Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 18:28:08 +0200
Subject: [PATCH 01/48] [wip] Describe actionmap

---
 data/actionsmap/yunohost.yml | 84 ++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 8509bfb23..1a234b9b0 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -920,6 +920,90 @@ backup:
                     extra:
                         pattern: *pattern_backup_archive_name
 
+    subcategories:
+        repository:
+            subcategory_help: Manage backup repositories
+            actions:
+                
+                ### backup_repository_list()
+                list:
+                    action_help: List available repositories where put archives
+                    api: GET /backup/repositories
+                    arguments:
+                        name:
+                            help: Name of the repository
+                            extra:
+                                pattern: *pattern_backup_repo_name
+
+                ### backup_repository_info()
+                info:
+                    action_help: Show info about a repository
+                    api: GET /backup/repositories
+                    arguments:
+                        -H:
+                            full: --human-readable
+                            help: Print sizes in human readable format
+                            action: store_true
+                        --space-used:
+                            help: Display size used
+                            action: store_true
+                
+                ### backup_repository_add()
+                add:
+                    action_help: Add a backup repository
+                    api: POST /backup/repository/<name>
+                    configuration:
+                        authenticate: all
+                        authenticator: ldap-anonymous
+                    arguments:
+                        path:
+                            help: Path eventually on another server
+                            extra:
+                                pattern: *pattern_backup_repo_path
+                        -n:
+                            full: --name
+                            help: Name of the repository
+                            extra:
+                                pattern: *pattern_backup_repo_name
+                        -d:
+                            full: --description
+                            help: Short description of the repository
+                        --methods:
+                            help: List of backup methods accepted
+                            nargs: "*"
+                        -q:
+                            full: --quota
+                            help: Quota to configure with this repository
+                        --no-encryption:
+                            help: If distant don't encrypt 
+
+                ### backup_repository_update()
+                update:
+                    action_help: Update a backup repository
+                    api: PUT /backup/repository/<name>
+                    arguments:
+                        -d:
+                            full: --description
+                            help: Short description of the repository
+                        -q:
+                            full: --quota
+                            help: Quota to configure with this repository
+                        -p:
+                            full: --password
+                            help: Change password
+                            extra:
+                                password: ask__password
+                                pattern: *pattern_password
+                
+                ### backup_repository_remove()
+                remove:
+                    action_help: Remove a backup repository
+                    api: DELETE /backup/repository/<name>
+                    arguments:
+                        name:
+                            help: Name of the backup repository to remove
+                            extra:
+                                pattern: *pattern_backup_repo_name
 
 #############################
 #          Monitor          #

From c52922d6da9285b951b266037e02573a49bddcba Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 18:37:28 +0200
Subject: [PATCH 02/48] [fix] Bad command description

---
 data/actionsmap/yunohost.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 1a234b9b0..9565fefa3 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -929,12 +929,7 @@ backup:
                 list:
                     action_help: List available repositories where put archives
                     api: GET /backup/repositories
-                    arguments:
-                        name:
-                            help: Name of the repository
-                            extra:
-                                pattern: *pattern_backup_repo_name
-
+                
                 ### backup_repository_info()
                 info:
                     action_help: Show info about a repository

From 74efa794f8c743a6111bc205c89543255fe6f1e2 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 18:46:04 +0200
Subject: [PATCH 03/48] [wip] Repository functions descriptors

---
 src/yunohost/repository.py | 90 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 src/yunohost/repository.py

diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
new file mode 100644
index 000000000..e0d976a2d
--- /dev/null
+++ b/src/yunohost/repository.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2013 YunoHost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+""" yunohost_repository.py
+
+    Manage backup repositories
+"""
+import os
+import re
+import json
+import errno
+import time
+import tarfile
+import shutil
+import subprocess
+
+from moulinette import msignals, m18n
+from moulinette.core import MoulinetteError
+from moulinette.utils import filesystem
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_file
+
+from yunohost.monitor import binary_to_human
+from yunohost.log import OperationLogger
+
+BACKUP_PATH = '/home/yunohost.backup'
+ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
+logger = getActionLogger('yunohost.backup')
+
+
+def backup_repository_list(name):
+    """
+    List available repositories where put archives
+    """
+    pass
+
+def backup_repository_info(name):
+    """
+    Show info about a repository
+
+    Keyword arguments:
+        name -- Name of the backup repository
+    """
+    pass
+
+def backup_repository_add(name):
+    """
+    Add a backup repository
+
+    Keyword arguments:
+        name -- Name of the backup repository
+    """
+    pass
+
+def backup_repository_update(name):
+    """
+    Update a backup repository
+
+    Keyword arguments:
+        name -- Name of the backup repository
+    """
+    pass
+
+def backup_repository_remove(name):
+    """
+    Remove a backup repository
+
+    Keyword arguments:
+        name -- Name of the backup repository to remove
+
+    """
+    pass

From 621bffbfbe2d1964780fdd13dfa6f0f8d2cd8201 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 18:56:26 +0200
Subject: [PATCH 04/48] [wip] Add args and disable space used by default

---
 data/actionsmap/yunohost.yml | 17 +++++++++++------
 src/yunohost/repository.py   |  7 ++++---
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 9565fefa3..f7131eddf 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -941,7 +941,7 @@ backup:
                             action: store_true
                         --space-used:
                             help: Display size used
-                            action: store_true
+                            action: store_false
                 
                 ### backup_repository_add()
                 add:
@@ -954,12 +954,12 @@ backup:
                         path:
                             help: Path eventually on another server
                             extra:
-                                pattern: *pattern_backup_repo_path
+                                pattern: *pattern_backup_repository_path
                         -n:
                             full: --name
                             help: Name of the repository
                             extra:
-                                pattern: *pattern_backup_repo_name
+                                pattern: *pattern_backup_repository_name
                         -d:
                             full: --description
                             help: Short description of the repository
@@ -969,14 +969,19 @@ backup:
                         -q:
                             full: --quota
                             help: Quota to configure with this repository
-                        --no-encryption:
-                            help: If distant don't encrypt 
+                        --e:
+                            full: --encryption
+                            help: Type of encryption
 
                 ### backup_repository_update()
                 update:
                     action_help: Update a backup repository
                     api: PUT /backup/repository/<name>
                     arguments:
+                        name:
+                            help: Name of the backup repository to update
+                            extra:
+                                pattern: *pattern_backup_repository_name
                         -d:
                             full: --description
                             help: Short description of the repository
@@ -998,7 +1003,7 @@ backup:
                         name:
                             help: Name of the backup repository to remove
                             extra:
-                                pattern: *pattern_backup_repo_name
+                                pattern: *pattern_backup_repository_name
 
 #############################
 #          Monitor          #
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index e0d976a2d..e3e53eb24 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -52,7 +52,7 @@ def backup_repository_list(name):
     """
     pass
 
-def backup_repository_info(name):
+def backup_repository_info(name, human_readble=True, space_used=False):
     """
     Show info about a repository
 
@@ -61,7 +61,8 @@ def backup_repository_info(name):
     """
     pass
 
-def backup_repository_add(name):
+def backup_repository_add(name, path, name, description=None, methods=None,
+                          quota=None, encryption="passphrase"):
     """
     Add a backup repository
 
@@ -70,7 +71,7 @@ def backup_repository_add(name):
     """
     pass
 
-def backup_repository_update(name):
+def backup_repository_update(name, description=None, quota=None, password=None):
     """
     Update a backup repository
 

From e973259561e663874a92ca89b8ab914c8715f771 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 18:57:29 +0200
Subject: [PATCH 05/48] [wip] Typo

---
 src/yunohost/repository.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index e3e53eb24..45cb51aab 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -52,7 +52,7 @@ def backup_repository_list(name):
     """
     pass
 
-def backup_repository_info(name, human_readble=True, space_used=False):
+def backup_repository_info(name, human_readable=True, space_used=False):
     """
     Show info about a repository
 

From 25aeef36f9e59defc8c36caa4fda9de0a8a638b8 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 19:52:39 +0200
Subject: [PATCH 06/48] [wip] Create repositories.yml file

---
 src/yunohost/repository.py | 128 ++++++++++++++++++++++++++++++++++---
 1 file changed, 118 insertions(+), 10 deletions(-)

diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 45cb51aab..c102708f1 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -44,13 +44,13 @@ from yunohost.log import OperationLogger
 BACKUP_PATH = '/home/yunohost.backup'
 ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
 logger = getActionLogger('yunohost.backup')
-
+REPOSITORIES_PATH = '/etc/yunohost/repositories.yml'
 
 def backup_repository_list(name):
     """
     List available repositories where put archives
     """
-    pass
+    return _get_repositories()
 
 def backup_repository_info(name, human_readable=True, space_used=False):
     """
@@ -59,28 +59,87 @@ def backup_repository_info(name, human_readable=True, space_used=False):
     Keyword arguments:
         name -- Name of the backup repository
     """
-    pass
+    repositories = _get_repositories()
 
-def backup_repository_add(name, path, name, description=None, methods=None,
-                          quota=None, encryption="passphrase"):
+    if key not in repositories:
+        raise MoulinetteError(errno.EINVAL, m18n.n(
+            'backup_repository_doesnt_exists', name=name))
+
+    if human_readable:
+        logger.info("--human-readbale option not yet implemented")
+
+    if space_used:
+        logger.info("--space-used option not yet implemented")
+
+    return repositories[name]
+
+@is_unit_operation()
+def backup_repository_add(operation_logger, path, name, description=None,
+                          methods=None, quota=None, encryption="passphrase"):
     """
     Add a backup repository
 
     Keyword arguments:
         name -- Name of the backup repository
     """
-    pass
+    repositories = _get_repositories()
 
-def backup_repository_update(name, description=None, quota=None, password=None):
+    if name in repositories:
+        raise MoulinetteError(errno.EIO, m18n.n('backup_repositories_already_exists', repositories=name))
+
+    repositories[name]= {
+        'path': path
+    }
+
+    if description is not None:
+        repositories[name]['description'] = description
+
+    if methods is not None:
+        repositories[name]['methods'] = methods
+
+    if quota is not None:
+        repositories[name]['quota'] = quota
+
+    if encryption is not None:
+        repositories[name]['encryption'] = encryption
+
+    try:
+        _save_repositories(repositories)
+    except:
+        # we'll get a logger.warning with more details in _save_services
+        raise MoulinetteError(errno.EIO, m18n.n('backup_repository_add_failed',
+                                                repository=name, path=path))
+
+    logger.success(m18n.n('backup_repository_added', repository=name, path=path))
+
+@is_unit_operation()
+def backup_repository_update(operation_logger, name, description=None,
+                             quota=None, password=None):
     """
     Update a backup repository
 
     Keyword arguments:
         name -- Name of the backup repository
     """
-    pass
+    repositories = _get_repositories()
 
-def backup_repository_remove(name):
+    if name not in repositories:
+        raise MoulinetteError(errno.EINVAL, m18n.n(
+            'backup_repository_doesnt_exists', name=name))
+
+    if description is not None:
+        repositories[name]['description'] = description
+
+    if quota is not None:
+        repositories[name]['quota'] = quota
+
+    _save_repositories(repositories)
+
+    logger.success(m18n.n('backup_repository_updated', repository=name,
+                          path=repository['path']))
+
+@is_unit_operation()
+def backup_repository_remove(operation_logger, name):
     """
     Remove a backup repository
 
@@ -88,4 +147,53 @@ def backup_repository_remove(name):
         name -- Name of the backup repository to remove
 
     """
-    pass
+    repositories = _get_repositories()
+
+    repository = repositories.pop(name)
+
+    if repository is None:
+        raise MoulinetteError(errno.EINVAL, m18n.n(
+            'backup_repository_doesnt_exists', name=name))
+
+    _save_repositories(repositories)
+
+    logger.success(m18n.n('backup_repository_removed', repository=name,
+                          path=repository['path']))
+
+
+def _save_repositories(repositories):
+    """
+    Save managed repositories to file
+
+    Keyword argument:
+        repositories -- A dict of managed repositories with their parameters
+
+    """
+    try:
+        write_to_json(REPOSITORIES_PATH, repositories)
+    except Exception as e:
+        raise MoulinetteError(1, m18n.n('backup_cant_save_repositories_file',
+                                        reason=e),
+                              exc_info=1)
+
+
+def _get_repositories():
+    """
+    Read repositories configuration from file
+
+    Keyword argument:
+        repositories -- A dict of managed repositories with their parameters
+
+    """
+    repositories = {}
+
+    if os.path.exists(REPOSITORIES_PATH):
+        try:
+            repositories = read_json(REPOSITORIES_PATH)
+        except MoulinetteError as e:
+            raise MoulinetteError(1,
+                                  m18n.n('backup_cant_open_repositories_file',
+                                         reason=e),
+                                  exc_info=1)
+
+    return repositories

From 9a6723dcb2d548deeaf27ec3f93045d3ff200c3d Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 20:05:14 +0200
Subject: [PATCH 07/48] [wip] Human readbale quota size

---
 src/yunohost/repository.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index c102708f1..a6c2d7839 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -41,9 +41,7 @@ from moulinette.utils.filesystem import read_file
 from yunohost.monitor import binary_to_human
 from yunohost.log import OperationLogger
 
-BACKUP_PATH = '/home/yunohost.backup'
-ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
-logger = getActionLogger('yunohost.backup')
+logger = getActionLogger('yunohost.repository')
 REPOSITORIES_PATH = '/etc/yunohost/repositories.yml'
 
 def backup_repository_list(name):
@@ -61,17 +59,22 @@ def backup_repository_info(name, human_readable=True, space_used=False):
     """
     repositories = _get_repositories()
 
-    if key not in repositories:
+    repository = repositories.pop(name, None)
+
+    if repository is None:
         raise MoulinetteError(errno.EINVAL, m18n.n(
             'backup_repository_doesnt_exists', name=name))
 
-    if human_readable:
-        logger.info("--human-readbale option not yet implemented")
-
     if space_used:
         logger.info("--space-used option not yet implemented")
 
-    return repositories[name]
+    if human_readable:
+        if 'quota' in repository:
+            repository['quota'] = binary_to_human(repository['quota'])
+        if 'used' in repository:
+            repository['used'] = binary_to_human(repository['used'])
+
+    return repository
 
 @is_unit_operation()
 def backup_repository_add(operation_logger, path, name, description=None,
@@ -106,7 +109,6 @@ def backup_repository_add(operation_logger, path, name, description=None,
     try:
         _save_repositories(repositories)
     except:
-        # we'll get a logger.warning with more details in _save_services
         raise MoulinetteError(errno.EIO, m18n.n('backup_repository_add_failed',
                                                 repository=name, path=path))
 

From 5f4fd01f626316b1651422512e2a1c0d61231a36 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 20:13:59 +0200
Subject: [PATCH 08/48] [wip] Only list name of repository by default

---
 data/actionsmap/yunohost.yml | 4 ++++
 src/yunohost/repository.py   | 9 +++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index f7131eddf..d301340fc 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -929,6 +929,10 @@ backup:
                 list:
                     action_help: List available repositories where put archives
                     api: GET /backup/repositories
+                    arguments:
+                        --full:
+                            help: Show more details
+                            action: store_true
                 
                 ### backup_repository_info()
                 info:
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index a6c2d7839..d07409d29 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -44,11 +44,16 @@ from yunohost.log import OperationLogger
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_PATH = '/etc/yunohost/repositories.yml'
 
-def backup_repository_list(name):
+def backup_repository_list(name, full=False):
     """
     List available repositories where put archives
     """
-    return _get_repositories()
+    repositories = _get_repositories()
+
+    if full:
+        return repositories
+    else:
+        return repositories.keys()
 
 def backup_repository_info(name, human_readable=True, space_used=False):
     """

From 62229d52640724eb860a491720010fe654e3eab7 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 20:15:06 +0200
Subject: [PATCH 09/48] [fix] Info repository end points

---
 data/actionsmap/yunohost.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index d301340fc..2e1b5b831 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -937,7 +937,7 @@ backup:
                 ### backup_repository_info()
                 info:
                     action_help: Show info about a repository
-                    api: GET /backup/repositories
+                    api: GET /backup/repository
                     arguments:
                         -H:
                             full: --human-readable

From 3d4ec54b5f54f7783d5547d7d0fcde78a2381798 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 20:29:36 +0200
Subject: [PATCH 10/48] [fix] Purge option on removing repositories

---
 data/actionsmap/yunohost.yml |  3 +++
 src/yunohost/repository.py   | 20 +++++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 2e1b5b831..2ccc64243 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -1008,6 +1008,9 @@ backup:
                             help: Name of the backup repository to remove
                             extra:
                                 pattern: *pattern_backup_repository_name
+                        --purge:
+                            help: Remove all archives and data inside repository
+                            action: store_false
 
 #############################
 #          Monitor          #
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index d07409d29..d3212b5ba 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -71,12 +71,13 @@ def backup_repository_info(name, human_readable=True, space_used=False):
             'backup_repository_doesnt_exists', name=name))
 
     if space_used:
-        logger.info("--space-used option not yet implemented")
+        try:
+            repository['used'] = _get_repository_used_space(name)
 
     if human_readable:
         if 'quota' in repository:
             repository['quota'] = binary_to_human(repository['quota'])
-        if 'used' in repository:
+        if 'used' in repository and isinstance(repository['used', int):
             repository['used'] = binary_to_human(repository['used'])
 
     return repository
@@ -146,7 +147,7 @@ def backup_repository_update(operation_logger, name, description=None,
                           path=repository['path']))
 
 @is_unit_operation()
-def backup_repository_remove(operation_logger, name):
+def backup_repository_remove(operation_logger, name, purge=False):
     """
     Remove a backup repository
 
@@ -204,3 +205,16 @@ def _get_repositories():
                                   exc_info=1)
 
     return repositories
+
+
+def _get_repository_used_space(path, methods=None):
+    """
+    Return the used space on a repository or 'unknown' if method don't support
+    this feature
+
+    Keyword argument:
+        path -- Path of the repository
+
+    """
+    logger.info("--space-used option not yet implemented")
+    return 'unknown'

From 077eb3fc344f3a37efbc4716f5ac9ff452ab5623 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 31 Aug 2018 20:31:35 +0200
Subject: [PATCH 11/48] [fix] Remove try managed by another way

---
 src/yunohost/repository.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index d3212b5ba..f6e01b601 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -71,8 +71,7 @@ def backup_repository_info(name, human_readable=True, space_used=False):
             'backup_repository_doesnt_exists', name=name))
 
     if space_used:
-        try:
-            repository['used'] = _get_repository_used_space(name)
+        repository['used'] = _get_repository_used_space(name)
 
     if human_readable:
         if 'quota' in repository:

From 164362c8b3486acd6064131197ea7c6a44dd06ef Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Sat, 1 Sep 2018 02:48:27 +0200
Subject: [PATCH 12/48] [wip] Add class BackupRepository

---
 src/yunohost/repository.py | 224 ++++++++++++++++++++-----------------
 1 file changed, 121 insertions(+), 103 deletions(-)

diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index f6e01b601..21be1d92e 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -44,11 +44,111 @@ from yunohost.log import OperationLogger
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_PATH = '/etc/yunohost/repositories.yml'
 
+class BackupRepository(object):
+
+    @classmethod
+    def get(cls, name):
+        cls.load()
+
+        if name not in cls.repositories:
+            raise MoulinetteError(errno.EINVAL, m18n.n(
+                'backup_repository_doesnt_exists', name=name))
+
+        return BackupRepository(**repositories[name])
+
+    @classmethod
+    def load(cls):
+        """
+        Read repositories configuration from file
+        """
+        cls.repositories = {}
+
+        if os.path.exists(REPOSITORIES_PATH):
+            try:
+                cls.repositories = read_json(REPOSITORIES_PATH)
+            except MoulinetteError as e:
+                raise MoulinetteError(1,
+                                      m18n.n('backup_cant_open_repositories_file',
+                                             reason=e),
+                                      exc_info=1)
+        return cls.repositories
+
+    @classmethod
+    def save(cls):
+        """
+        Save managed repositories to file
+        """
+        try:
+            write_to_json(REPOSITORIES_PATH, cls.repositories)
+        except Exception as e:
+            raise MoulinetteError(1, m18n.n('backup_cant_save_repositories_file',
+                                            reason=e),
+                                  exc_info=1)
+
+
+    def __init__(self, location, name, description=None, method=None,
+                 encryption=None, quota=None):
+
+        self.location = None
+        self._split_location()
+
+        self.name = location if name is None else name
+        if self.name in repositories:
+            raise MoulinetteError(errno.EIO, m18n.n('backup_repository_already_exists', repositories=name))
+
+        self.description = None
+        self.encryption = None
+        self.quota = None
+
+        if method is None:
+            method = 'tar' if self.domain is None else 'borg'
+        self.method = BackupMethod.create(method, self)
+
+    def compute_space_used(self):
+        if self.used is None:
+            try:
+                self.used = self.method.compute_space_used()
+            except NotYetImplemented as e:
+                self.used = 'unknown'
+        return self.used
+
+    def purge(self):
+        self.method.purge()
+
+    def delete(self, purge=False):
+        repositories = BackupRepositories.repositories
+
+        repository = repositories.pop(name)
+
+        BackupRepository.save()
+
+        if purge:
+            self.purge()
+
+    def save(self):
+        BackupRepository.reposirories[self.name] = self.__dict__
+        BackupRepository.save()
+
+    def _split_location(self):
+        """
+        Split a repository location into protocol, user, domain and path
+        """
+        location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+:))?(?P<path>[^\0]+)$'
+        location_match = re.match(location_regex, self.location)
+
+        if location_match is None:
+            raise MoulinetteError(errno.EIO, m18n.n('backup_repositories_invalid_location', location=location))
+
+        self.protocol = location_match.group('protocol')
+        self.user = location_match.group('user')
+        self.domain = location_match.group('domain')
+        self.path = location_match.group('path')
+
 def backup_repository_list(name, full=False):
     """
     List available repositories where put archives
     """
-    repositories = _get_repositories()
+    repositories = BackupRepository.load()
 
     if full:
         return repositories
@@ -62,17 +162,13 @@ def backup_repository_info(name, human_readable=True, space_used=False):
     Keyword arguments:
         name -- Name of the backup repository
     """
-    repositories = _get_repositories()
+    repository = BackupRepository.get(name)
 
-    repository = repositories.pop(name, None)
-
-    if repository is None:
-        raise MoulinetteError(errno.EINVAL, m18n.n(
-            'backup_repository_doesnt_exists', name=name))
 
     if space_used:
-        repository['used'] = _get_repository_used_space(name)
+        repository.compute_space_used()
 
+    repository = repository.__dict__
     if human_readable:
         if 'quota' in repository:
             repository['quota'] = binary_to_human(repository['quota'])
@@ -82,7 +178,7 @@ def backup_repository_info(name, human_readable=True, space_used=False):
     return repository
 
 @is_unit_operation()
-def backup_repository_add(operation_logger, path, name, description=None,
+def backup_repository_add(operation_logger, location, name, description=None,
                           methods=None, quota=None, encryption="passphrase"):
     """
     Add a backup repository
@@ -90,34 +186,15 @@ def backup_repository_add(operation_logger, path, name, description=None,
     Keyword arguments:
         name -- Name of the backup repository
     """
-    repositories = _get_repositories()
-
-    if name in repositories:
-        raise MoulinetteError(errno.EIO, m18n.n('backup_repositories_already_exists', repositories=name))
-
-    repositories[name]= {
-        'path': path
-    }
-
-    if description is not None:
-        repositories[name]['description'] = description
-
-    if methods is not None:
-        repositories[name]['methods'] = methods
-
-    if quota is not None:
-        repositories[name]['quota'] = quota
-
-    if encryption is not None:
-        repositories[name]['encryption'] = encryption
+    repository = BackupRepository(location, name, description, methods, quota, encryption)
 
     try:
-        _save_repositories(repositories)
-    except:
+        repository.save()
+    except MoulinetteError as e:
         raise MoulinetteError(errno.EIO, m18n.n('backup_repository_add_failed',
-                                                repository=name, path=path))
+                                                repository=name, location=location))
 
-    logger.success(m18n.n('backup_repository_added', repository=name, path=path))
+    logger.success(m18n.n('backup_repository_added', repository=name, location=location))
 
 @is_unit_operation()
 def backup_repository_update(operation_logger, name, description=None,
@@ -128,22 +205,21 @@ def backup_repository_update(operation_logger, name, description=None,
     Keyword arguments:
         name -- Name of the backup repository
     """
-    repositories = _get_repositories()
-
-    if name not in repositories:
-        raise MoulinetteError(errno.EINVAL, m18n.n(
-            'backup_repository_doesnt_exists', name=name))
+    repository = BackupRepository.get(name)
 
     if description is not None:
-        repositories[name]['description'] = description
+        repository.description = description
 
     if quota is not None:
-        repositories[name]['quota'] = quota
-
-    _save_repositories(repositories)
+        repository.quota = quota
 
+    try:
+        repository.save()
+    except MoulinetteError as e:
+        raise MoulinetteError(errno.EIO, m18n.n('backup_repository_update_failed',
+                                                repository=name))
     logger.success(m18n.n('backup_repository_updated', repository=name,
-                          path=repository['path']))
+                          location=repository['location']))
 
 @is_unit_operation()
 def backup_repository_remove(operation_logger, name, purge=False):
@@ -154,66 +230,8 @@ def backup_repository_remove(operation_logger, name, purge=False):
         name -- Name of the backup repository to remove
 
     """
-    repositories = _get_repositories()
-
-    repository = repositories.pop(name)
-
-    if repository is None:
-        raise MoulinetteError(errno.EINVAL, m18n.n(
-            'backup_repository_doesnt_exists', name=name))
-
-    _save_repositories(repositories)
-
+    repository = BackupRepository.get(name)
+    repository.delete(purge)
     logger.success(m18n.n('backup_repository_removed', repository=name,
                           path=repository['path']))
 
-
-def _save_repositories(repositories):
-    """
-    Save managed repositories to file
-
-    Keyword argument:
-        repositories -- A dict of managed repositories with their parameters
-
-    """
-    try:
-        write_to_json(REPOSITORIES_PATH, repositories)
-    except Exception as e:
-        raise MoulinetteError(1, m18n.n('backup_cant_save_repositories_file',
-                                        reason=e),
-                              exc_info=1)
-
-
-def _get_repositories():
-    """
-    Read repositories configuration from file
-
-    Keyword argument:
-        repositories -- A dict of managed repositories with their parameters
-
-    """
-    repositories = {}
-
-    if os.path.exists(REPOSITORIES_PATH):
-        try:
-            repositories = read_json(REPOSITORIES_PATH)
-        except MoulinetteError as e:
-            raise MoulinetteError(1,
-                                  m18n.n('backup_cant_open_repositories_file',
-                                         reason=e),
-                                  exc_info=1)
-
-    return repositories
-
-
-def _get_repository_used_space(path, methods=None):
-    """
-    Return the used space on a repository or 'unknown' if method don't support
-    this feature
-
-    Keyword argument:
-        path -- Path of the repository
-
-    """
-    logger.info("--space-used option not yet implemented")
-    return 'unknown'

From e96d76814ffdaf7d5b83f8abbfbd1918d3004f0d Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 28 Dec 2018 15:17:55 +0100
Subject: [PATCH 13/48] [wip] Pep8 + repo change

---
 src/yunohost/backup.py     | 45 +++---------------
 src/yunohost/repository.py | 96 ++++++++++++++++++++++----------------
 2 files changed, 63 insertions(+), 78 deletions(-)

diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index 745291fb1..83e3168c1 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -71,7 +71,6 @@ class BackupRestoreTargetsManager(object):
     """
 
     def __init__(self):
-
         self.targets = {}
         self.results = {
             "system": {},
@@ -1969,8 +1968,7 @@ class CustomBackupMethod(BackupMethod):
 # "Front-end"                                                               #
 #
 
-def backup_create(name=None, description=None, methods=[],
-                  output_directory=None, no_compress=False,
+def backup_create(name=None, description=None, repos=[],
                   system=[], apps=[]):
     """
     Create a backup local archive
@@ -1995,30 +1993,6 @@ def backup_create(name=None, description=None, methods=[],
     if name and name in backup_list()['archives']:
         raise YunohostError('backup_archive_name_exists')
 
-    # Validate output_directory option
-    if output_directory:
-        output_directory = os.path.abspath(output_directory)
-
-        # Check for forbidden folders
-        if output_directory.startswith(ARCHIVES_PATH) or \
-            re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
-                     output_directory):
-            raise YunohostError('backup_output_directory_forbidden')
-
-        # Check that output directory is empty
-        if os.path.isdir(output_directory) and no_compress and \
-                os.listdir(output_directory):
-            raise YunohostError('backup_output_directory_not_empty')
-    elif no_compress:
-        raise YunohostError('backup_output_directory_required')
-
-    # Define methods (retro-compat)
-    if not methods:
-        if no_compress:
-            methods = ['copy']
-        else:
-            methods = ['tar']  # In future, borg will be the default actions
-
     # If no --system or --apps given, backup everything
     if system is None and apps is None:
         system = []
@@ -2032,20 +2006,15 @@ def backup_create(name=None, description=None, methods=[],
     _create_archive_dir()
 
     # Prepare files to backup
-    if no_compress:
-        backup_manager = BackupManager(name, description,
-                                       work_dir=output_directory)
-    else:
-        backup_manager = BackupManager(name, description)
+    backup_manager = BackupManager(name, description)
 
     # Add backup methods
-    if output_directory:
-        methods = BackupMethod.create(methods, output_directory)
-    else:
-        methods = BackupMethod.create(methods)
+    if repos == []:
+        repos = ['/home/yunohost.backup/archives']
 
-    for method in methods:
-        backup_manager.add(method)
+    methods = []
+    for repo in repos:
+        backup_manager.add(BackupMethod.create(methods, repo))
 
     # Add backup targets (system and apps)
     backup_manager.set_system_targets(system)
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 21be1d92e..348262907 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -2,7 +2,7 @@
 
 """ License
 
-    Copyright (C) 2013 YunoHost
+    Copyright (C) 2013 Yunohost
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU Affero General Public License as published
@@ -25,36 +25,39 @@
 """
 import os
 import re
-import json
-import errno
 import time
-import tarfile
-import shutil
 import subprocess
 
 from moulinette import msignals, m18n
 from moulinette.core import MoulinetteError
 from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file
+from moulinette.utils.filesystem import read_file, read_json, write_to_json
 
+
+from yunohost.utils.error import YunohostError
 from yunohost.monitor import binary_to_human
-from yunohost.log import OperationLogger
+from yunohost.log import OperationLogger, is_unit_operation
+from yunohost.backup import BackupMethod
 
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_PATH = '/etc/yunohost/repositories.yml'
 
+
 class BackupRepository(object):
+    """
+    BackupRepository manage all repository the admin added to the instance
+    """
+    repositories = {}
 
     @classmethod
     def get(cls, name):
         cls.load()
 
         if name not in cls.repositories:
-            raise MoulinetteError(errno.EINVAL, m18n.n(
-                'backup_repository_doesnt_exists', name=name))
+            raise YunohostError('backup_repository_doesnt_exists', name=name)
 
-        return BackupRepository(**repositories[name])
+        return BackupRepository(**cls.repositories[name])
 
     @classmethod
     def load(cls):
@@ -67,10 +70,7 @@ class BackupRepository(object):
             try:
                 cls.repositories = read_json(REPOSITORIES_PATH)
             except MoulinetteError as e:
-                raise MoulinetteError(1,
-                                      m18n.n('backup_cant_open_repositories_file',
-                                             reason=e),
-                                      exc_info=1)
+                raise YunohostError('backup_cant_open_repositories_file', reason=e)
         return cls.repositories
 
     @classmethod
@@ -81,34 +81,41 @@ class BackupRepository(object):
         try:
             write_to_json(REPOSITORIES_PATH, cls.repositories)
         except Exception as e:
-            raise MoulinetteError(1, m18n.n('backup_cant_save_repositories_file',
-                                            reason=e),
-                                  exc_info=1)
+            raise YunohostError('backup_cant_save_repositories_file', reason=e)
 
-
-    def __init__(self, location, name, description=None, method=None,
+    def __init__(self, location, name=None, description=None, method=None,
                  encryption=None, quota=None):
 
-        self.location = None
+        self.location = location
         self._split_location()
 
         self.name = location if name is None else name
-        if self.name in repositories:
-            raise MoulinetteError(errno.EIO, m18n.n('backup_repository_already_exists', repositories=name))
+        if self.name in BackupMethod.repositories:
+            raise YunohostError('backup_repository_already_exists', repositories=name)
 
-        self.description = None
-        self.encryption = None
-        self.quota = None
+        self.description = description
+        self.encryption = encryption
+        self.quota = quota
 
         if method is None:
             method = 'tar' if self.domain is None else 'borg'
         self.method = BackupMethod.create(method, self)
+        
+        # Check for forbidden folders
+        if self.path.startswith(ARCHIVES_PATH) or \
+            re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
+                     self.path):
+            raise YunohostError('backup_output_directory_forbidden')
+
+        # Check that output directory is empty
+        if os.path.isdir(location) and os.listdir(location):
+            raise YunohostError('backup_output_directory_not_empty')
 
     def compute_space_used(self):
         if self.used is None:
             try:
                 self.used = self.method.compute_space_used()
-            except NotYetImplemented as e:
+            except (AttributeError, NotImplementedError):
                 self.used = 'unknown'
         return self.used
 
@@ -116,9 +123,9 @@ class BackupRepository(object):
         self.method.purge()
 
     def delete(self, purge=False):
-        repositories = BackupRepositories.repositories
+        repositories = BackupRepository.repositories
 
-        repository = repositories.pop(name)
+        repositories.pop(self.name)
 
         BackupRepository.save()
 
@@ -137,13 +144,15 @@ class BackupRepository(object):
         location_match = re.match(location_regex, self.location)
 
         if location_match is None:
-            raise MoulinetteError(errno.EIO, m18n.n('backup_repositories_invalid_location', location=location))
+            raise YunohostError('backup_repositories_invalid_location', 
+                                location=location)
 
         self.protocol = location_match.group('protocol')
         self.user = location_match.group('user')
         self.domain = location_match.group('domain')
         self.path = location_match.group('path')
 
+
 def backup_repository_list(name, full=False):
     """
     List available repositories where put archives
@@ -155,6 +164,7 @@ def backup_repository_list(name, full=False):
     else:
         return repositories.keys()
 
+
 def backup_repository_info(name, human_readable=True, space_used=False):
     """
     Show info about a repository
@@ -164,7 +174,6 @@ def backup_repository_info(name, human_readable=True, space_used=False):
     """
     repository = BackupRepository.get(name)
 
-
     if space_used:
         repository.compute_space_used()
 
@@ -172,11 +181,12 @@ def backup_repository_info(name, human_readable=True, space_used=False):
     if human_readable:
         if 'quota' in repository:
             repository['quota'] = binary_to_human(repository['quota'])
-        if 'used' in repository and isinstance(repository['used', int):
+        if 'used' in repository and isinstance(repository['used'], int):
             repository['used'] = binary_to_human(repository['used'])
 
     return repository
 
+
 @is_unit_operation()
 def backup_repository_add(operation_logger, location, name, description=None,
                           methods=None, quota=None, encryption="passphrase"):
@@ -184,17 +194,24 @@ def backup_repository_add(operation_logger, location, name, description=None,
     Add a backup repository
 
     Keyword arguments:
+        location -- Location of the repository (could be a remote location)
         name -- Name of the backup repository
+        description -- An optionnal description
+        quota -- Maximum size quota of the repository
+        encryption -- If available, the kind of encryption to use
     """
-    repository = BackupRepository(location, name, description, methods, quota, encryption)
+    repository = BackupRepository(
+        location, name, description, methods, quota, encryption)
 
     try:
         repository.save()
-    except MoulinetteError as e:
-        raise MoulinetteError(errno.EIO, m18n.n('backup_repository_add_failed',
-                                                repository=name, location=location))
+    except MoulinetteError:
+        raise YunohostError('backup_repository_add_failed',
+                            repository=name, location=location)
+
+    logger.success(m18n.n('backup_repository_added',
+                          repository=name, location=location))
 
-    logger.success(m18n.n('backup_repository_added', repository=name, location=location))
 
 @is_unit_operation()
 def backup_repository_update(operation_logger, name, description=None,
@@ -215,12 +232,12 @@ def backup_repository_update(operation_logger, name, description=None,
 
     try:
         repository.save()
-    except MoulinetteError as e:
-        raise MoulinetteError(errno.EIO, m18n.n('backup_repository_update_failed',
-                                                repository=name))
+    except MoulinetteError:
+        raise YunohostError('backup_repository_update_failed', repository=name)
     logger.success(m18n.n('backup_repository_updated', repository=name,
                           location=repository['location']))
 
+
 @is_unit_operation()
 def backup_repository_remove(operation_logger, name, purge=False):
     """
@@ -234,4 +251,3 @@ def backup_repository_remove(operation_logger, name, purge=False):
     repository.delete(purge)
     logger.success(m18n.n('backup_repository_removed', repository=name,
                           path=repository['path']))
-

From 48f7e51dd880d202d4eca0a88c5b7e586496aa7f Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 28 Dec 2018 20:29:59 +0100
Subject: [PATCH 14/48] [wip] One BackupMethod by BackupRepo

---
 src/yunohost/repository.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 348262907..56893c1a0 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -50,6 +50,12 @@ class BackupRepository(object):
     """
     repositories = {}
 
+    @classmethod
+    def create(cls, location, name, *args, **kwargs):
+        cls.load()
+
+        return BackupRepository(True, location, name, *args, **kwargs)
+
     @classmethod
     def get(cls, name):
         cls.load()
@@ -57,14 +63,15 @@ class BackupRepository(object):
         if name not in cls.repositories:
             raise YunohostError('backup_repository_doesnt_exists', name=name)
 
-        return BackupRepository(**cls.repositories[name])
+        return BackupRepository(False, **cls.repositories[name])
 
     @classmethod
     def load(cls):
         """
         Read repositories configuration from file
         """
-        cls.repositories = {}
+        if cls.repositories != {}:
+            return cls.repositories
 
         if os.path.exists(REPOSITORIES_PATH):
             try:
@@ -83,15 +90,15 @@ class BackupRepository(object):
         except Exception as e:
             raise YunohostError('backup_cant_save_repositories_file', reason=e)
 
-    def __init__(self, location, name=None, description=None, method=None,
+    def __init__(self, created=True, location, name=None, description=None, method=None,
                  encryption=None, quota=None):
 
         self.location = location
         self._split_location()
 
         self.name = location if name is None else name
-        if self.name in BackupMethod.repositories:
-            raise YunohostError('backup_repository_already_exists', repositories=name)
+        if created and self.name in BackupMethod.repositories:
+            raise YunohostError('backup_repository_already_exists', repositories=self.name)
 
         self.description = description
         self.encryption = encryption
@@ -99,17 +106,11 @@ class BackupRepository(object):
 
         if method is None:
             method = 'tar' if self.domain is None else 'borg'
-        self.method = BackupMethod.create(method, self)
-        
-        # Check for forbidden folders
-        if self.path.startswith(ARCHIVES_PATH) or \
-            re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
-                     self.path):
-            raise YunohostError('backup_output_directory_forbidden')
+        if created:
+            self.method = BackupMethod.create(method, self)
+        else:
+            self.method = BackupMethod.get(method, self)
 
-        # Check that output directory is empty
-        if os.path.isdir(location) and os.listdir(location):
-            raise YunohostError('backup_output_directory_not_empty')
 
     def compute_space_used(self):
         if self.used is None:

From a7af79d93e0b3e4971041c27219603dc22854296 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 1 Jan 2019 16:24:08 +0100
Subject: [PATCH 15/48] [wip] Support local repository

---
 src/yunohost/backup.py     |  30 +++---
 src/yunohost/repository.py | 214 ++++++++++++++++++-------------------
 2 files changed, 123 insertions(+), 121 deletions(-)

diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index 83e3168c1..687d02258 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -24,7 +24,6 @@
     Manage backups
 """
 import os
-import re
 import json
 import time
 import tarfile
@@ -52,6 +51,7 @@ from yunohost.monitor import binary_to_human
 from yunohost.tools import tools_postinstall
 from yunohost.service import service_regen_conf
 from yunohost.log import OperationLogger
+from yunohost.repository import BackupRepository
 from functools import reduce
 
 BACKUP_PATH = '/home/yunohost.backup'
@@ -1425,7 +1425,9 @@ class BackupMethod(object):
                    BackupRepository object. If None, the default repo is used :
                    /home/yunohost.backup/archives/
         """
-        self.repo = ARCHIVES_PATH if repo is None else repo
+        if not repo or isinstance(repo, basestring):
+            repo = BackupRepository.get_or_create(ARCHIVES_PATH)
+        self.repo = repo
 
     @property
     def method_name(self):
@@ -1596,7 +1598,7 @@ class BackupMethod(object):
                 try:
                     subprocess.check_call(["mount", "--rbind", src, dest])
                     subprocess.check_call(["mount", "-o", "remount,ro,bind", dest])
-                except Exception as e:
+                except Exception:
                     logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest))
                     # To check if dest is mounted, use /proc/mounts that
                     # escape spaces as \040
@@ -1697,6 +1699,7 @@ class CopyBackupMethod(BackupMethod):
 
     def __init__(self, repo=None):
         super(CopyBackupMethod, self).__init__(repo)
+        filesystem.mkdir(self.repo.path, parent=True)
 
     @property
     def method_name(self):
@@ -1709,7 +1712,7 @@ class CopyBackupMethod(BackupMethod):
 
         for path in self.manager.paths_to_backup:
             source = path['source']
-            dest = os.path.join(self.repo, path['dest'])
+            dest = os.path.join(self.repo.path, path['dest'])
             if source == dest:
                 logger.debug("Files already copyed")
                 return
@@ -1735,18 +1738,18 @@ class CopyBackupMethod(BackupMethod):
         # the ynh cli
         super(CopyBackupMethod, self).mount()
 
-        if not os.path.isdir(self.repo):
+        if not os.path.isdir(self.repo.path):
             raise YunohostError('backup_no_uncompress_archive_dir')
 
         filesystem.mkdir(self.work_dir, parent=True)
-        ret = subprocess.call(["mount", "-r", "--rbind", self.repo,
+        ret = subprocess.call(["mount", "-r", "--rbind", self.repo.path,
                               self.work_dir])
         if ret == 0:
             return
         else:
             logger.warning(m18n.n("bind_mouting_disable"))
-            subprocess.call(["mountpoint", "-q", dest,
-                            "&&", "umount", "-R", dest])
+            subprocess.call(["mountpoint", "-q", self.repo.path,
+                            "&&", "umount", "-R", self.repo.path])
             raise YunohostError('backup_cant_mount_uncompress_archive')
 
 
@@ -1758,6 +1761,7 @@ class TarBackupMethod(BackupMethod):
 
     def __init__(self, repo=None):
         super(TarBackupMethod, self).__init__(repo)
+        filesystem.mkdir(self.repo.path, parent=True)
 
     @property
     def method_name(self):
@@ -1766,7 +1770,7 @@ class TarBackupMethod(BackupMethod):
     @property
     def _archive_file(self):
         """Return the compress archive path"""
-        return os.path.join(self.repo, self.name + '.tar.gz')
+        return os.path.join(self.repo.path, self.name + '.tar.gz')
 
     def backup(self):
         """
@@ -1781,8 +1785,8 @@ class TarBackupMethod(BackupMethod):
                                          compress archive
         """
 
-        if not os.path.exists(self.repo):
-            filesystem.mkdir(self.repo, 0o750, parents=True, uid='admin')
+        if not os.path.exists(self.repo.path):
+            filesystem.mkdir(self.repo.path, 0o750, parents=True, uid='admin')
 
         # Check free space in output
         self._check_is_enough_free_space()
@@ -2012,9 +2016,9 @@ def backup_create(name=None, description=None, repos=[],
     if repos == []:
         repos = ['/home/yunohost.backup/archives']
 
-    methods = []
     for repo in repos:
-        backup_manager.add(BackupMethod.create(methods, repo))
+        repo = BackupRepository.get(repo)
+        backup_manager.add(repo.method)
 
     # Add backup targets (system and apps)
     backup_manager.set_system_targets(system)
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 56893c1a0..2b3fce6cd 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -77,7 +77,8 @@ class BackupRepository(object):
             try:
                 cls.repositories = read_json(REPOSITORIES_PATH)
             except MoulinetteError as e:
-                raise YunohostError('backup_cant_open_repositories_file', reason=e)
+                raise YunohostError(
+                    'backup_cant_open_repositories_file', reason=e)
         return cls.repositories
 
     @classmethod
@@ -98,7 +99,8 @@ class BackupRepository(object):
 
         self.name = location if name is None else name
         if created and self.name in BackupMethod.repositories:
-            raise YunohostError('backup_repository_already_exists', repositories=self.name)
+            raise YunohostError(
+                'backup_repository_already_exists', repositories=self.name)
 
         self.description = description
         self.encryption = encryption
@@ -106,11 +108,7 @@ class BackupRepository(object):
 
         if method is None:
             method = 'tar' if self.domain is None else 'borg'
-        if created:
-            self.method = BackupMethod.create(method, self)
-        else:
-            self.method = BackupMethod.get(method, self)
-
+        self.method = BackupMethod.get(method, self)
 
     def compute_space_used(self):
         if self.used is None:
@@ -128,127 +126,127 @@ class BackupRepository(object):
 
         repositories.pop(self.name)
 
-        BackupRepository.save()
+            BackupRepository.save()
 
-        if purge:
-            self.purge()
+            if purge:
+                self.purge()
 
-    def save(self):
-        BackupRepository.reposirories[self.name] = self.__dict__
-        BackupRepository.save()
+        def save(self):
+            BackupRepository.reposirories[self.name] = self.__dict__
+            BackupRepository.save()
 
-    def _split_location(self):
+        def _split_location(self):
+            """
+            Split a repository location into protocol, user, domain and path
+            """
+            location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+:))?(?P<path>[^\0]+)$'
+            location_match = re.match(location_regex, self.location)
+
+            if location_match is None:
+                raise YunohostError('backup_repositories_invalid_location',
+                                    location=location)
+
+            self.protocol = location_match.group('protocol')
+            self.user = location_match.group('user')
+            self.domain = location_match.group('domain')
+            self.path = location_match.group('path')
+
+
+    def backup_repository_list(name, full=False):
         """
-        Split a repository location into protocol, user, domain and path
+        List available repositories where put archives
         """
-        location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+:))?(?P<path>[^\0]+)$'
-        location_match = re.match(location_regex, self.location)
+        repositories = BackupRepository.load()
 
-        if location_match is None:
-            raise YunohostError('backup_repositories_invalid_location', 
-                                location=location)
-
-        self.protocol = location_match.group('protocol')
-        self.user = location_match.group('user')
-        self.domain = location_match.group('domain')
-        self.path = location_match.group('path')
+        if full:
+            return repositories
+        else:
+            return repositories.keys()
 
 
-def backup_repository_list(name, full=False):
-    """
-    List available repositories where put archives
-    """
-    repositories = BackupRepository.load()
+    def backup_repository_info(name, human_readable=True, space_used=False):
+        """
+        Show info about a repository
 
-    if full:
-        return repositories
-    else:
-        return repositories.keys()
+        Keyword arguments:
+            name -- Name of the backup repository
+        """
+        repository = BackupRepository.get(name)
+
+        if space_used:
+            repository.compute_space_used()
+
+        repository = repository.__dict__
+        if human_readable:
+            if 'quota' in repository:
+                repository['quota'] = binary_to_human(repository['quota'])
+            if 'used' in repository and isinstance(repository['used'], int):
+                repository['used'] = binary_to_human(repository['used'])
+
+        return repository
 
 
-def backup_repository_info(name, human_readable=True, space_used=False):
-    """
-    Show info about a repository
+    @is_unit_operation()
+    def backup_repository_add(operation_logger, location, name, description=None,
+                            methods=None, quota=None, encryption="passphrase"):
+        """
+        Add a backup repository
 
-    Keyword arguments:
-        name -- Name of the backup repository
-    """
-    repository = BackupRepository.get(name)
+        Keyword arguments:
+            location -- Location of the repository (could be a remote location)
+            name -- Name of the backup repository
+            description -- An optionnal description
+            quota -- Maximum size quota of the repository
+            encryption -- If available, the kind of encryption to use
+        """
+        repository = BackupRepository(
+            location, name, description, methods, quota, encryption)
 
-    if space_used:
-        repository.compute_space_used()
+        try:
+            repository.save()
+        except MoulinetteError:
+            raise YunohostError('backup_repository_add_failed',
+                                repository=name, location=location)
 
-    repository = repository.__dict__
-    if human_readable:
-        if 'quota' in repository:
-            repository['quota'] = binary_to_human(repository['quota'])
-        if 'used' in repository and isinstance(repository['used'], int):
-            repository['used'] = binary_to_human(repository['used'])
-
-    return repository
+        logger.success(m18n.n('backup_repository_added',
+                            repository=name, location=location))
 
 
-@is_unit_operation()
-def backup_repository_add(operation_logger, location, name, description=None,
-                          methods=None, quota=None, encryption="passphrase"):
-    """
-    Add a backup repository
+    @is_unit_operation()
+    def backup_repository_update(operation_logger, name, description=None,
+                                quota=None, password=None):
+        """
+        Update a backup repository
 
-    Keyword arguments:
-        location -- Location of the repository (could be a remote location)
-        name -- Name of the backup repository
-        description -- An optionnal description
-        quota -- Maximum size quota of the repository
-        encryption -- If available, the kind of encryption to use
-    """
-    repository = BackupRepository(
-        location, name, description, methods, quota, encryption)
+        Keyword arguments:
+            name -- Name of the backup repository
+        """
+        repository = BackupRepository.get(name)
 
-    try:
-        repository.save()
-    except MoulinetteError:
-        raise YunohostError('backup_repository_add_failed',
-                            repository=name, location=location)
+        if description is not None:
+            repository.description = description
 
-    logger.success(m18n.n('backup_repository_added',
-                          repository=name, location=location))
+        if quota is not None:
+            repository.quota = quota
+
+        try:
+            repository.save()
+        except MoulinetteError:
+            raise YunohostError('backup_repository_update_failed', repository=name)
+        logger.success(m18n.n('backup_repository_updated', repository=name,
+                            location=repository['location']))
 
 
-@is_unit_operation()
-def backup_repository_update(operation_logger, name, description=None,
-                             quota=None, password=None):
-    """
-    Update a backup repository
+    @is_unit_operation()
+    def backup_repository_remove(operation_logger, name, purge=False):
+        """
+        Remove a backup repository
 
-    Keyword arguments:
-        name -- Name of the backup repository
-    """
-    repository = BackupRepository.get(name)
+        Keyword arguments:
+            name -- Name of the backup repository to remove
 
-    if description is not None:
-        repository.description = description
-
-    if quota is not None:
-        repository.quota = quota
-
-    try:
-        repository.save()
-    except MoulinetteError:
-        raise YunohostError('backup_repository_update_failed', repository=name)
-    logger.success(m18n.n('backup_repository_updated', repository=name,
-                          location=repository['location']))
-
-
-@is_unit_operation()
-def backup_repository_remove(operation_logger, name, purge=False):
-    """
-    Remove a backup repository
-
-    Keyword arguments:
-        name -- Name of the backup repository to remove
-
-    """
-    repository = BackupRepository.get(name)
-    repository.delete(purge)
-    logger.success(m18n.n('backup_repository_removed', repository=name,
-                          path=repository['path']))
+        """
+        repository = BackupRepository.get(name)
+        repository.delete(purge)
+        logger.success(m18n.n('backup_repository_removed', repository=name,
+                            path=repository['path']))

From 9f5b826078288a370bf794247d81361c1712fd0d Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 1 Jan 2019 16:42:48 +0100
Subject: [PATCH 16/48] [wip] Work on borg local repo

---
 src/yunohost/backup.py     |  26 ++++-
 src/yunohost/repository.py | 208 ++++++++++++++++++-------------------
 2 files changed, 128 insertions(+), 106 deletions(-)

diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index 687d02258..62c3e32ce 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -1888,13 +1888,37 @@ class TarBackupMethod(BackupMethod):
 
 class BorgBackupMethod(BackupMethod):
 
+    def __init__(self, repo=None):
+        super(TarBackupMethod, self).__init__(repo)
+        if not self.repo.domain:
+            filesystem.mkdir(self.repo.path, parent=True)
+        else:
+            #Todo Initialize remote repo
+            pass
+
     @property
     def method_name(self):
         return 'borg'
 
     def backup(self):
         """ Backup prepared files with borg """
-        super(CopyBackupMethod, self).backup()
+        super(BorgBackupMethod, self).backup()
+        
+        for path in self.manager.paths_to_backup:
+            source = path['source']
+            dest = os.path.join(self.repo.path, path['dest'])
+            if source == dest:
+                logger.debug("Files already copyed")
+                return
+
+            dest_parent = os.path.dirname(dest)
+            if not os.path.exists(dest_parent):
+                filesystem.mkdir(dest_parent, 0o750, True, uid='admin')
+
+            if os.path.isdir(source):
+                shutil.copytree(source, dest)
+            else:
+                shutil.copy(source, dest)
 
         # TODO run borg create command
         raise YunohostError('backup_borg_not_implemented')
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 2b3fce6cd..08cabb42a 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -91,7 +91,7 @@ class BackupRepository(object):
         except Exception as e:
             raise YunohostError('backup_cant_save_repositories_file', reason=e)
 
-    def __init__(self, created=True, location, name=None, description=None, method=None,
+    def __init__(self, created, location, name=None, description=None, method=None,
                  encryption=None, quota=None):
 
         self.location = location
@@ -126,127 +126,125 @@ class BackupRepository(object):
 
         repositories.pop(self.name)
 
-            BackupRepository.save()
+        BackupRepository.save()
 
-            if purge:
-                self.purge()
+        if purge:
+            self.purge()
 
-        def save(self):
-            BackupRepository.reposirories[self.name] = self.__dict__
-            BackupRepository.save()
+    def save(self):
+        BackupRepository.reposirories[self.name] = self.__dict__
+        BackupRepository.save()
 
-        def _split_location(self):
-            """
-            Split a repository location into protocol, user, domain and path
-            """
-            location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+:))?(?P<path>[^\0]+)$'
-            location_match = re.match(location_regex, self.location)
-
-            if location_match is None:
-                raise YunohostError('backup_repositories_invalid_location',
-                                    location=location)
-
-            self.protocol = location_match.group('protocol')
-            self.user = location_match.group('user')
-            self.domain = location_match.group('domain')
-            self.path = location_match.group('path')
-
-
-    def backup_repository_list(name, full=False):
+    def _split_location(self):
         """
-        List available repositories where put archives
+        Split a repository location into protocol, user, domain and path
         """
-        repositories = BackupRepository.load()
+        location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+:))?(?P<path>[^\0]+)$'
+        location_match = re.match(location_regex, self.location)
 
-        if full:
-            return repositories
-        else:
-            return repositories.keys()
+        if location_match is None:
+            raise YunohostError('backup_repositories_invalid_location',
+                                location=location)
+
+        self.protocol = location_match.group('protocol')
+        self.user = location_match.group('user')
+        self.domain = location_match.group('domain')
+        self.path = location_match.group('path')
+
+def backup_repository_list(name, full=False):
+    """
+    List available repositories where put archives
+    """
+    repositories = BackupRepository.load()
+
+    if full:
+        return repositories
+    else:
+        return repositories.keys()
+
+def backup_repository_info(name, human_readable=True, space_used=False):
+    """
+    Show info about a repository
+
+    Keyword arguments:
+        name -- Name of the backup repository
+    """
+    repository = BackupRepository.get(name)
+
+    if space_used:
+        repository.compute_space_used()
+
+    repository = repository.__dict__
+    if human_readable:
+        if 'quota' in repository:
+            repository['quota'] = binary_to_human(repository['quota'])
+        if 'used' in repository and isinstance(repository['used'], int):
+            repository['used'] = binary_to_human(repository['used'])
+
+    return repository
 
 
-    def backup_repository_info(name, human_readable=True, space_used=False):
-        """
-        Show info about a repository
+@is_unit_operation()
+def backup_repository_add(operation_logger, location, name, description=None,
+                        methods=None, quota=None, encryption="passphrase"):
+    """
+    Add a backup repository
 
-        Keyword arguments:
-            name -- Name of the backup repository
-        """
-        repository = BackupRepository.get(name)
+    Keyword arguments:
+        location -- Location of the repository (could be a remote location)
+        name -- Name of the backup repository
+        description -- An optionnal description
+        quota -- Maximum size quota of the repository
+        encryption -- If available, the kind of encryption to use
+    """
+    repository = BackupRepository(
+        location, name, description, methods, quota, encryption)
 
-        if space_used:
-            repository.compute_space_used()
+    try:
+        repository.save()
+    except MoulinetteError:
+        raise YunohostError('backup_repository_add_failed',
+                            repository=name, location=location)
 
-        repository = repository.__dict__
-        if human_readable:
-            if 'quota' in repository:
-                repository['quota'] = binary_to_human(repository['quota'])
-            if 'used' in repository and isinstance(repository['used'], int):
-                repository['used'] = binary_to_human(repository['used'])
-
-        return repository
+    logger.success(m18n.n('backup_repository_added',
+                        repository=name, location=location))
 
 
-    @is_unit_operation()
-    def backup_repository_add(operation_logger, location, name, description=None,
-                            methods=None, quota=None, encryption="passphrase"):
-        """
-        Add a backup repository
+@is_unit_operation()
+def backup_repository_update(operation_logger, name, description=None,
+                            quota=None, password=None):
+    """
+    Update a backup repository
 
-        Keyword arguments:
-            location -- Location of the repository (could be a remote location)
-            name -- Name of the backup repository
-            description -- An optionnal description
-            quota -- Maximum size quota of the repository
-            encryption -- If available, the kind of encryption to use
-        """
-        repository = BackupRepository(
-            location, name, description, methods, quota, encryption)
+    Keyword arguments:
+        name -- Name of the backup repository
+    """
+    repository = BackupRepository.get(name)
 
-        try:
-            repository.save()
-        except MoulinetteError:
-            raise YunohostError('backup_repository_add_failed',
-                                repository=name, location=location)
+    if description is not None:
+        repository.description = description
 
-        logger.success(m18n.n('backup_repository_added',
-                            repository=name, location=location))
+    if quota is not None:
+        repository.quota = quota
+
+    try:
+        repository.save()
+    except MoulinetteError:
+        raise YunohostError('backup_repository_update_failed', repository=name)
+    logger.success(m18n.n('backup_repository_updated', repository=name,
+                        location=repository['location']))
 
 
-    @is_unit_operation()
-    def backup_repository_update(operation_logger, name, description=None,
-                                quota=None, password=None):
-        """
-        Update a backup repository
+@is_unit_operation()
+def backup_repository_remove(operation_logger, name, purge=False):
+    """
+    Remove a backup repository
 
-        Keyword arguments:
-            name -- Name of the backup repository
-        """
-        repository = BackupRepository.get(name)
+    Keyword arguments:
+        name -- Name of the backup repository to remove
 
-        if description is not None:
-            repository.description = description
-
-        if quota is not None:
-            repository.quota = quota
-
-        try:
-            repository.save()
-        except MoulinetteError:
-            raise YunohostError('backup_repository_update_failed', repository=name)
-        logger.success(m18n.n('backup_repository_updated', repository=name,
-                            location=repository['location']))
-
-
-    @is_unit_operation()
-    def backup_repository_remove(operation_logger, name, purge=False):
-        """
-        Remove a backup repository
-
-        Keyword arguments:
-            name -- Name of the backup repository to remove
-
-        """
-        repository = BackupRepository.get(name)
-        repository.delete(purge)
-        logger.success(m18n.n('backup_repository_removed', repository=name,
-                            path=repository['path']))
+    """
+    repository = BackupRepository.get(name)
+    repository.delete(purge)
+    logger.success(m18n.n('backup_repository_removed', repository=name,
+                        path=repository['path']))

From d43a86b13662e1e749f1e21e7abf4e500e15d806 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 2 Jan 2019 19:27:58 +0100
Subject: [PATCH 17/48] [wip] Remote borg repository

---
 data/actionsmap/yunohost.yml |  4 ++
 src/yunohost/backup.py       | 79 +++++++++++++++++++++++++-----------
 2 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index a0b2447ce..9f1652335 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -891,6 +891,10 @@ backup:
             action_help: List available local backup archives
             api: GET /backup/archives
             arguments:
+                -r:
+                    full: --repos
+                    help: List archives in these repositories
+                    nargs: "*"
                 -i:
                     full: --with-info
                     help: Show backup information for each archive
diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index 62c3e32ce..054e6ac37 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -1889,42 +1889,75 @@ class TarBackupMethod(BackupMethod):
 class BorgBackupMethod(BackupMethod):
 
     def __init__(self, repo=None):
-        super(TarBackupMethod, self).__init__(repo)
+        super(BorgBackupMethod, self).__init__(repo)
+
         if not self.repo.domain:
             filesystem.mkdir(self.repo.path, parent=True)
-        else:
-            #Todo Initialize remote repo
-            pass
+        
+        cmd = ['borg', 'init', self.repo.location]
+
+        if self.repo.quota:
+            cmd += ['--storage-quota', self.repo.quota]
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError('backup_borg_init_error')
+
 
     @property
     def method_name(self):
         return 'borg'
+    
+    def need_mount(self):
+        return True
 
     def backup(self):
         """ Backup prepared files with borg """
-        super(BorgBackupMethod, self).backup()
+
+        archive = self.repo.location + '::' + self.name
+        cmd = ['borg', 'create', archive, './']
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError('backup_borg_mount_error')
+
+    def mount(self, restore_manager):
+        """ Extract and mount needed files with borg """
+        super(BorgBackupMethod, self).mount(restore_manager)
         
-        for path in self.manager.paths_to_backup:
-            source = path['source']
-            dest = os.path.join(self.repo.path, path['dest'])
-            if source == dest:
-                logger.debug("Files already copyed")
-                return
+        # Export as tar needed files through a pipe
+        archive = self.repo.location + '::' + self.name
+        cmd = ['borg', 'export-tar', archive, '-']
+        borg = self._run_borg_command(cmd, stdout=subprocess.PIPE)
 
-            dest_parent = os.path.dirname(dest)
-            if not os.path.exists(dest_parent):
-                filesystem.mkdir(dest_parent, 0o750, True, uid='admin')
+        # And uncompress it into the working directory
+        untar = subprocess.Popen(['tar', 'x'], cwd=self.work_dir, stdin=borg.stdout)
+        borg_return_code = borg.wait()
+        untar_return_code = untar.wait()
+        if borg_return_code + untar_return_code != 0:
+            err = untar.communicate()[1]
+            raise YunohostError('backup_borg_backup_error')
 
-            if os.path.isdir(source):
-                shutil.copytree(source, dest)
-            else:
-                shutil.copy(source, dest)
+    def _run_borg_command(self, cmd, stdout=None):
+        env = dict(os.environ)
 
-        # TODO run borg create command
-        raise YunohostError('backup_borg_not_implemented')
+        if self.repo.domain:
+            # TODO Use the best/good key
+            private_key = "/root/.ssh/ssh_host_ed25519_key"
+            
+            # Don't check ssh fingerprint strictly the first time
+            # TODO improve this by publishing and checking this with DNS
+            strict = 'yes' if self.repo.domain in open('/root/.ssh/known_hosts').read() else 'no'
+            env['BORG_RSH'] = "ssh -i %s -oStrictHostKeyChecking=%s"
+            env['BORG_RSH'] = env['BORG_RSH'] % (private_key, strict)
+        
+        # In case, borg need a passphrase to get access to the repo
+        if self.repo.passphrase:
+            cmd += ['-e', 'repokey']
+            env['BORG_PASSPHRASE'] = self.repo.passphrase
+
+        return subprocess.Popen(cmd, env=env, stdout=stdout)
 
-    def mount(self, mnt_path):
-        raise YunohostError('backup_borg_not_implemented')
 
 
 class CustomBackupMethod(BackupMethod):
@@ -1943,7 +1976,7 @@ class CustomBackupMethod(BackupMethod):
 
     @property
     def method_name(self):
-        return 'borg'
+        return 'custom'
 
     def need_mount(self):
         """Call the backup_method hook to know if we need to organize files

From 1219583692b147c710e5d259b3fac6382485a468 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 4 Jan 2019 17:39:05 +0100
Subject: [PATCH 18/48] [wip] Be able to list and get info with tar and borg

---
 src/yunohost/backup.py     | 252 ++++++++++++++++++++++++-------------
 src/yunohost/repository.py |   3 +
 2 files changed, 168 insertions(+), 87 deletions(-)

diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index 054e6ac37..1a74405fc 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -1403,6 +1403,7 @@ class BackupMethod(object):
         mount_and_backup(self, backup_manager)
         mount(self, restore_manager)
         create(cls, method, **kwargs)
+        info(archive_name)
 
     Usage:
         method = BackupMethod.create("tar")
@@ -1504,6 +1505,20 @@ class BackupMethod(object):
         """
         self.manager = restore_manager
 
+    def info(self, name):
+        self._assert_archive_exists()
+           
+        info_json = self._get_info_string()
+        if not self._info_json:
+            raise YunohostError('backup_info_json_not_implemented')
+        try:
+            info = json.load(info_json)
+        except:
+            logger.debug("unable to load info json", exc_info=1)
+            raise YunohostError('backup_invalid_archive')
+        
+        return info
+    
     def clean(self):
         """
         Umount sub directories of working dirextories and delete it if temporary
@@ -1768,7 +1783,7 @@ class TarBackupMethod(BackupMethod):
         return 'tar'
 
     @property
-    def _archive_file(self):
+    def archive_path(self):
         """Return the compress archive path"""
         return os.path.join(self.repo.path, self.name + '.tar.gz')
 
@@ -1793,10 +1808,10 @@ class TarBackupMethod(BackupMethod):
 
         # Open archive file for writing
         try:
-            tar = tarfile.open(self._archive_file, "w:gz")
+            tar = tarfile.open(self.archive_path, "w:gz")
         except:
             logger.debug("unable to open '%s' for writing",
-                         self._archive_file, exc_info=1)
+                         self.archive_path, exc_info=1)
             raise YunohostError('backup_archive_open_failed')
 
         # Add files to the archive
@@ -1812,13 +1827,13 @@ class TarBackupMethod(BackupMethod):
 
         # Move info file
         shutil.copy(os.path.join(self.work_dir, 'info.json'),
-                    os.path.join(ARCHIVES_PATH, self.name + '.info.json'))
+                    os.path.join(self.repo.path, self.name + '.info.json'))
 
         # If backuped to a non-default location, keep a symlink of the archive
         # to that location
-        link = os.path.join(ARCHIVES_PATH, self.name + '.tar.gz')
+        link = os.path.join(self.repo.path, self.name + '.tar.gz')
         if not os.path.isfile(link):
-            os.symlink(self._archive_file, link)
+            os.symlink(self.archive_path, link)
 
     def mount(self, restore_manager):
         """
@@ -1829,19 +1844,22 @@ class TarBackupMethod(BackupMethod):
         backup_archive_open_failed -- Raised if the archive can't be open
         """
         super(TarBackupMethod, self).mount(restore_manager)
+        
+        # Check file exist and it's not a broken link
+        self._assert_archive_exists()
 
         # Check the archive can be open
         try:
-            tar = tarfile.open(self._archive_file, "r:gz")
+            tar = tarfile.open(self.archive_path, "r:gz")
         except:
             logger.debug("cannot open backup archive '%s'",
-                         self._archive_file, exc_info=1)
+                         self.archive_path, exc_info=1)
             raise YunohostError('backup_archive_open_failed')
         tar.close()
 
         # Mount the tarball
         logger.debug(m18n.n("restore_extracting"))
-        tar = tarfile.open(self._archive_file, "r:gz")
+        tar = tarfile.open(self.archive_path, "r:gz")
         tar.extract('info.json', path=self.work_dir)
 
         try:
@@ -1885,6 +1903,66 @@ class TarBackupMethod(BackupMethod):
             ]
             tar.extractall(members=subdir_and_files, path=self.work_dir)
 
+    def list(self):
+        result = []
+
+        try:
+            # Retrieve local archives
+            archives = os.listdir(self.repo.path)
+        except OSError:
+            logger.debug("unable to iterate over local archives", exc_info=1)
+        else:
+            # Iterate over local archives
+            for f in archives:
+                try:
+                    name = f[:f.rindex('.tar.gz')]
+                except ValueError:
+                    continue
+                result.append(name)
+            result.sort(key=lambda x: os.path.getctime(self.archive_path))
+
+        return result
+
+    def _archive_exists(self):
+        return os.path.lexists(self.archive_path)
+    
+    def _assert_archive_exists(self):
+        if not self._archive_exists():
+            raise YunohostError('backup_archive_name_unknown', name=self.name)
+
+        # If symlink, retrieve the real path
+        if os.path.islink(self.archive_path):
+            archive_file = os.path.realpath(self.archive_path)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+                raise YunohostError('backup_archive_broken_link',
+                                    path=archive_file)
+
+    def _get_info_string(self):
+        info_file = "%s/%s.info.json" % (self.repo.path, self.name)
+
+        if not os.path.exists(info_file):
+            tar = tarfile.open(self.archive_path, "r:gz")
+            info_dir = info_file + '.d'
+            try:
+                tar.extract('info.json', path=info_dir)
+            except KeyError:
+                logger.debug("unable to retrieve '%s' inside the archive",
+                             info_file, exc_info=1)
+                raise YunohostError('backup_invalid_archive')
+            else:
+                shutil.move(os.path.join(info_dir, 'info.json'), info_file)
+            finally:
+                tar.close()
+            os.rmdir(info_dir)
+
+        try:
+            return read_file(info_file)
+        except MoulinetteError:
+            logger.debug("unable to load '%s'", info_file, exc_info=1)
+            raise YunohostError('backup_invalid_archive')
+
 
 class BorgBackupMethod(BackupMethod):
 
@@ -1908,14 +1986,18 @@ class BorgBackupMethod(BackupMethod):
     def method_name(self):
         return 'borg'
     
+    @property
+    def archive_path(self):
+        """Return the archive path"""
+        return self.repo.location + '::' + self.name
+    
     def need_mount(self):
         return True
 
     def backup(self):
         """ Backup prepared files with borg """
 
-        archive = self.repo.location + '::' + self.name
-        cmd = ['borg', 'create', archive, './']
+        cmd = ['borg', 'create', self.archive_path, './']
         borg = self._run_borg_command(cmd)
         return_code = borg.wait()
         if return_code:
@@ -1926,8 +2008,7 @@ class BorgBackupMethod(BackupMethod):
         super(BorgBackupMethod, self).mount(restore_manager)
         
         # Export as tar needed files through a pipe
-        archive = self.repo.location + '::' + self.name
-        cmd = ['borg', 'export-tar', archive, '-']
+        cmd = ['borg', 'export-tar', self.archive_path, '-']
         borg = self._run_borg_command(cmd, stdout=subprocess.PIPE)
 
         # And uncompress it into the working directory
@@ -1938,6 +2019,38 @@ class BorgBackupMethod(BackupMethod):
             err = untar.communicate()[1]
             raise YunohostError('backup_borg_backup_error')
 
+    def list(self):
+        cmd = ['borg', 'list', self.repo.location, '--short']
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError('backup_borg_list_error')
+        
+        out, _ = borg.communicate()
+        
+        result = out.strip().splitlines()
+
+        return result
+    
+    def _assert_archive_exists(self):
+        cmd = ['borg', 'list', self.archive_path]
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError('backup_borg_archive_name_unknown')
+
+    def _get_info_string(self):
+        # Export as tar info file through a pipe
+        cmd = ['borg', 'extract', '--stdout', self.archive_path, 'info.json']
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError('backup_borg_info_error')
+        
+        out, _ = borg.communicate()
+
+        return out
+
     def _run_borg_command(self, cmd, stdout=None):
         env = dict(os.environ)
 
@@ -2169,46 +2282,43 @@ def backup_restore(auth, name, system=[], apps=[], force=False):
     return restore_manager.targets.results
 
 
-def backup_list(with_info=False, human_readable=False):
+def backup_list(repos=[], with_info=False, human_readable=False):
     """
     List available local backup archives
 
     Keyword arguments:
+        repos -- Repositories from which list archives
         with_info -- Show backup information for each archive
         human_readable -- Print sizes in human readable format
 
     """
-    result = []
-
-    try:
-        # Retrieve local archives
-        archives = os.listdir(ARCHIVES_PATH)
-    except OSError:
-        logger.debug("unable to iterate over local archives", exc_info=1)
+    result = OrderedDict()
+    
+    if repos == []:
+        repos = BackupRepository.all()
     else:
-        # Iterate over local archives
-        for f in archives:
-            try:
-                name = f[:f.rindex('.tar.gz')]
-            except ValueError:
-                continue
-            result.append(name)
-        result.sort(key=lambda x: os.path.getctime(os.path.join(ARCHIVES_PATH, x + ".tar.gz")))
-
-    if result and with_info:
-        d = OrderedDict()
-        for a in result:
-            try:
-                d[a] = backup_info(a, human_readable=human_readable)
-            except YunohostError as e:
-                logger.warning('%s: %s' % (a, e.strerror))
-
-        result = d
-
-    return {'archives': result}
+        for k, repo in repos:
+            repos[k] = BackupRepository.get(repo)
 
 
-def backup_info(name, with_details=False, human_readable=False):
+    for repo in repos:
+        result[repo.name] = repo.list(with_info)
+        
+        # Add details
+        if result[repo.name] and with_info:
+            d = OrderedDict()
+            for a in result[repo.name]:
+                try:
+                    d[a] = backup_info(a, repo=repo.location, human_readable=human_readable)
+                except YunohostError as e:
+                    logger.warning('%s: %s' % (a, e.strerror))
+
+            result[repo.name] = d
+    
+    return result
+
+
+def backup_info(name, repo=None, with_details=False, human_readable=False):
     """
     Get info about a local backup archive
 
@@ -2218,62 +2328,29 @@ def backup_info(name, with_details=False, human_readable=False):
         human_readable -- Print sizes in human readable format
 
     """
-    archive_file = '%s/%s.tar.gz' % (ARCHIVES_PATH, name)
+    if not repo:
+        repo = '/home/yunohost.backup/archives/'
 
-    # Check file exist (even if it's a broken symlink)
-    if not os.path.lexists(archive_file):
-        raise YunohostError('backup_archive_name_unknown', name=name)
+    repo = BackupRepository.get(repo)
 
-    # If symlink, retrieve the real path
-    if os.path.islink(archive_file):
-        archive_file = os.path.realpath(archive_file)
-
-        # Raise exception if link is broken (e.g. on unmounted external storage)
-        if not os.path.exists(archive_file):
-            raise YunohostError('backup_archive_broken_link',
-                                path=archive_file)
-
-    info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name)
-
-    if not os.path.exists(info_file):
-        tar = tarfile.open(archive_file, "r:gz")
-        info_dir = info_file + '.d'
-        try:
-            tar.extract('info.json', path=info_dir)
-        except KeyError:
-            logger.debug("unable to retrieve '%s' inside the archive",
-                         info_file, exc_info=1)
-            raise YunohostError('backup_invalid_archive')
-        else:
-            shutil.move(os.path.join(info_dir, 'info.json'), info_file)
-        finally:
-            tar.close()
-        os.rmdir(info_dir)
-
-    try:
-        with open(info_file) as f:
-            # Retrieve backup info
-            info = json.load(f)
-    except:
-        logger.debug("unable to load '%s'", info_file, exc_info=1)
-        raise YunohostError('backup_invalid_archive')
-
-    # Retrieve backup size
+    info = repo.info(name)
+    
+    # Historically backup size was not here, in that case we know it's a tar archive
     size = info.get('size', 0)
     if not size:
-        tar = tarfile.open(archive_file, "r:gz")
+        tar = tarfile.open(repo.archive_path, "r:gz")
         size = reduce(lambda x, y: getattr(x, 'size', x) + getattr(y, 'size', y),
-                      tar.getmembers())
+                    tar.getmembers())
         tar.close()
-    if human_readable:
-        size = binary_to_human(size) + 'B'
-
+    
     result = {
-        'path': archive_file,
+        'path': repo.archive_path,
         'created_at': datetime.utcfromtimestamp(info['created_at']),
         'description': info['description'],
         'size': size,
     }
+    if human_readable:
+        result['size'] = binary_to_human(result['size']) + 'B'
 
     if with_details:
         system_key = "system"
@@ -2286,6 +2363,7 @@ def backup_info(name, with_details=False, human_readable=False):
     return result
 
 
+
 def backup_delete(name):
     """
     Delete a backup
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 08cabb42a..d7b4f9742 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -110,6 +110,9 @@ class BackupRepository(object):
             method = 'tar' if self.domain is None else 'borg'
         self.method = BackupMethod.get(method, self)
 
+    def list(self, with_info=False):
+        return self.method.list(with_info)
+
     def compute_space_used(self):
         if self.used is None:
             try:

From 78150938638392a5b1005c2f78eb99357d323ef7 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 4 Jan 2019 23:28:45 +0100
Subject: [PATCH 19/48] [wip] List archives of custom backup method

---
 src/yunohost/backup.py | 140 +++++++++++++++++++++++++----------------
 1 file changed, 87 insertions(+), 53 deletions(-)

diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index 1a74405fc..be94d7970 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -38,6 +38,7 @@ from collections import OrderedDict
 from moulinette import msignals, m18n
 from yunohost.utils.error import YunohostError
 from moulinette.utils import filesystem
+from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file
 
@@ -1435,6 +1436,11 @@ class BackupMethod(object):
         """Return the string name of a BackupMethod (eg "tar" or "copy")"""
         raise YunohostError('backup_abstract_method')
 
+    @property
+    def archive_path(self):
+        """Return the archive path"""
+        return self.repo.location + '::' + self.name
+    
     @property
     def name(self):
         """Return the backup name"""
@@ -1940,7 +1946,7 @@ class TarBackupMethod(BackupMethod):
                                     path=archive_file)
 
     def _get_info_string(self):
-        info_file = "%s/%s.info.json" % (self.repo.path, self.name)
+        info_file = os.path.join(self.repo.path, self.name + '.info.json')
 
         if not os.path.exists(info_file):
             tar = tarfile.open(self.archive_path, "r:gz")
@@ -1976,20 +1982,12 @@ class BorgBackupMethod(BackupMethod):
 
         if self.repo.quota:
             cmd += ['--storage-quota', self.repo.quota]
-        borg = self._run_borg_command(cmd)
-        return_code = borg.wait()
-        if return_code:
-            raise YunohostError('backup_borg_init_error')
-
+        self._call('init', cmd)
 
     @property
     def method_name(self):
         return 'borg'
     
-    @property
-    def archive_path(self):
-        """Return the archive path"""
-        return self.repo.location + '::' + self.name
     
     def need_mount(self):
         return True
@@ -1998,10 +1996,7 @@ class BorgBackupMethod(BackupMethod):
         """ Backup prepared files with borg """
 
         cmd = ['borg', 'create', self.archive_path, './']
-        borg = self._run_borg_command(cmd)
-        return_code = borg.wait()
-        if return_code:
-            raise YunohostError('backup_borg_mount_error')
+        self._call('backup', cmd)
 
     def mount(self, restore_manager):
         """ Extract and mount needed files with borg """
@@ -2016,42 +2011,41 @@ class BorgBackupMethod(BackupMethod):
         borg_return_code = borg.wait()
         untar_return_code = untar.wait()
         if borg_return_code + untar_return_code != 0:
-            err = untar.communicate()[1]
-            raise YunohostError('backup_borg_backup_error')
+            # err = untar.communicate()[1]
+            raise YunohostError('backup_borg_mount_error')
 
     def list(self):
-        cmd = ['borg', 'list', self.repo.location, '--short']
-        borg = self._run_borg_command(cmd)
-        return_code = borg.wait()
-        if return_code:
-            raise YunohostError('backup_borg_list_error')
-        
-        out, _ = borg.communicate()
-        
-        result = out.strip().splitlines()
+        """ Return a list of archives names
 
+        Exceptions:
+        backup_borg_list_error -- Raised if the borg script failed
+        """
+        cmd = ['borg', 'list', self.repo.location, '--short']
+        out = self._call('list', cmd)
+        result = out.strip().splitlines()
         return result
     
     def _assert_archive_exists(self):
+        """ Trigger an error if archive is missing
+
+        Exceptions:
+        backup_borg_exist_error -- Raised if the borg script failed
+        """
         cmd = ['borg', 'list', self.archive_path]
-        borg = self._run_borg_command(cmd)
-        return_code = borg.wait()
-        if return_code:
-            raise YunohostError('backup_borg_archive_name_unknown')
+        self._call('exist', cmd)
 
     def _get_info_string(self):
-        # Export as tar info file through a pipe
-        cmd = ['borg', 'extract', '--stdout', self.archive_path, 'info.json']
-        borg = self._run_borg_command(cmd)
-        return_code = borg.wait()
-        if return_code:
-            raise YunohostError('backup_borg_info_error')
-        
-        out, _ = borg.communicate()
+        """ Return json string of the info.json file
 
-        return out
+        Exceptions:
+        backup_borg_info_error -- Raised if the custom script failed
+        """
+        cmd = ['borg', 'extract', '--stdout', self.archive_path, 'info.json']
+        return self._call('info', cmd)
 
     def _run_borg_command(self, cmd, stdout=None):
+        """ Call a submethod of borg with the good context
+        """
         env = dict(os.environ)
 
         if self.repo.domain:
@@ -2070,6 +2064,16 @@ class BorgBackupMethod(BackupMethod):
             env['BORG_PASSPHRASE'] = self.repo.passphrase
 
         return subprocess.Popen(cmd, env=env, stdout=stdout)
+    
+    def _call(self, action, cmd):
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError('backup_borg_' + action + '_error')
+        
+        out, _ = borg.communicate()
+
+        return out
 
 
 
@@ -2101,9 +2105,9 @@ class CustomBackupMethod(BackupMethod):
             return self._need_mount
 
         ret = hook_callback('backup_method', [self.method],
-                            args=self._get_args('need_mount'))
+                            args=['need_mount'])
 
-        self._need_mount = True if ret['succeed'] else False
+        self._need_mount = bool(ret['succeed'])
         return self._need_mount
 
     def backup(self):
@@ -2114,10 +2118,8 @@ class CustomBackupMethod(BackupMethod):
         backup_custom_backup_error -- Raised if the custom script failed
         """
 
-        ret = hook_callback('backup_method', [self.method],
-                            args=self._get_args('backup'))
-        if ret['failed']:
-            raise YunohostError('backup_custom_backup_error')
+        self._call('backup', self.work_dir, self.name, self.repo.location, self.manager.size,
+                self.manager.description)
 
     def mount(self, restore_manager):
         """
@@ -2127,15 +2129,47 @@ class CustomBackupMethod(BackupMethod):
         backup_custom_mount_error -- Raised if the custom script failed
         """
         super(CustomBackupMethod, self).mount(restore_manager)
-        ret = hook_callback('backup_method', [self.method],
-                            args=self._get_args('mount'))
-        if ret['failed']:
-            raise YunohostError('backup_custom_mount_error')
+        self._call('mount', self.work_dir, self.name, self.repo.location, self.manager.size,
+                self.manager.description)
 
-    def _get_args(self, action):
-        """Return the arguments to give to the custom script"""
-        return [action, self.work_dir, self.name, self.repo, self.manager.size,
-                self.manager.description]
+    def list(self):
+        """ Return a list of archives names
+
+        Exceptions:
+        backup_custom_list_error -- Raised if the custom script failed
+        """
+        out = self._call('list', self.repo.location)
+        result = out.strip().splitlines()
+        return result
+    
+    def _assert_archive_exists(self):
+        """ Trigger an error if archive is missing
+
+        Exceptions:
+        backup_custom_exist_error -- Raised if the custom script failed
+        """
+        self._call('exist', self.name, self.repo.location)
+
+    def _get_info_string(self):
+        """ Return json string of the info.json file
+
+        Exceptions:
+        backup_custom_info_error -- Raised if the custom script failed
+        """
+        return self._call('info', self.name, self.repo.location)
+    
+    def _call(self, *args):
+        """ Call a submethod of backup method hook
+
+        Exceptions:
+        backup_custom_ACTION_error -- Raised if the custom script failed
+        """
+        ret = hook_callback('backup_method', [self.method],
+                            args=args)
+        if ret['failed']:
+            raise YunohostError('backup_custom_' + args[0] + '_error')
+
+        return ret['succeed'][self.method]['stdreturn']
 
 
 #
@@ -2333,7 +2367,7 @@ def backup_info(name, repo=None, with_details=False, human_readable=False):
 
     repo = BackupRepository.get(repo)
 
-    info = repo.info(name)
+    info = repo.method.info(name)
     
     # Historically backup size was not here, in that case we know it's a tar archive
     size = info.get('size', 0)

From 3ec43be650e10b8f8e8e8b18d4c7133766b2a520 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 16 Aug 2019 19:44:19 +0200
Subject: [PATCH 20/48] [fix] Be able to call some repositories actions

---
 data/actionsmap/yunohost.yml | 17 +++++++++--------
 src/yunohost/backup.py       | 33 +++++++++++++++++++++++++++++++++
 src/yunohost/repository.py   | 12 +++++++-----
 3 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 5bbd6e0dd..b5cc4c575 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -1039,19 +1039,20 @@ backup:
                 add:
                     action_help: Add a backup repository
                     api: POST /backup/repository/<name>
-                    configuration:
-                        authenticate: all
-                        authenticator: ldap-anonymous
                     arguments:
-                        path:
-                            help: Path eventually on another server
+                        location:
+                            help: Location on this server or on an other
                             extra:
-                                pattern: *pattern_backup_repository_path
+                                pattern: &pattern_backup_repository_location
+                                    - !!str ^((ssh://)?[a-z_]\w*@\[\w\-\.]+:)?(~?/)?[\w/]*$
+                                    - "pattern_backup_repository_location"
                         -n:
                             full: --name
                             help: Name of the repository
                             extra:
-                                pattern: *pattern_backup_repository_name
+                                pattern: &pattern_backup_repository_name
+                                    - !!str ^\w+$
+                                    - "pattern_backup_repository_name"
                         -d:
                             full: --description
                             help: Short description of the repository
@@ -1061,7 +1062,7 @@ backup:
                         -q:
                             full: --quota
                             help: Quota to configure with this repository
-                        --e:
+                        -e:
                             full: --encryption
                             help: Type of encryption
 
diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index d7a0fbafc..8f256491d 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -2571,6 +2571,39 @@ def backup_delete(name):
 
     logger.success(m18n.n('backup_deleted'))
 
+
+#
+# Repository subcategory
+#
+import yunohost.repository
+
+
+def backup_repository_list(full):
+    return yunohost.repository.backup_repository_list(full)
+
+
+def backup_repository_info(name, human_readable, space_used):
+    return yunohost.repository.backup_repository_info(name, human_readable, space_used)
+
+
+def backup_repository_add(location, name, description, methods, quota, encryption):
+    return yunohost.repository.backup_repository_add(location, name, description, methods, quota, encryption)
+
+
+def backup_repository_update(name, description, quota, password):
+    return yunohost.repository.backup_repository_update(name, description, quota, password)
+
+
+def backup_repository_remove(name, purge):
+    return yunohost.repository.backup_repository_remove(name, purge)
+
+
+
+#
+# End Repository subcategory
+#
+
+
 #
 # Misc helpers                                                              #
 #
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index d7b4f9742..cedafa8ae 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -32,13 +32,12 @@ from moulinette import msignals, m18n
 from moulinette.core import MoulinetteError
 from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, read_json, write_to_json
+from moulinette.utils.filesystem import read_file, read_yaml, write_to_json
 
 
 from yunohost.utils.error import YunohostError
 from yunohost.monitor import binary_to_human
 from yunohost.log import OperationLogger, is_unit_operation
-from yunohost.backup import BackupMethod
 
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_PATH = '/etc/yunohost/repositories.yml'
@@ -75,7 +74,7 @@ class BackupRepository(object):
 
         if os.path.exists(REPOSITORIES_PATH):
             try:
-                cls.repositories = read_json(REPOSITORIES_PATH)
+                cls.repositories = read_yaml(REPOSITORIES_PATH)['repositories']
             except MoulinetteError as e:
                 raise YunohostError(
                     'backup_cant_open_repositories_file', reason=e)
@@ -94,6 +93,7 @@ class BackupRepository(object):
     def __init__(self, created, location, name=None, description=None, method=None,
                  encryption=None, quota=None):
 
+        from yunohost.backup import BackupMethod
         self.location = location
         self._split_location()
 
@@ -154,7 +154,8 @@ class BackupRepository(object):
         self.domain = location_match.group('domain')
         self.path = location_match.group('path')
 
-def backup_repository_list(name, full=False):
+
+def backup_repository_list(full=False):
     """
     List available repositories where put archives
     """
@@ -165,6 +166,7 @@ def backup_repository_list(name, full=False):
     else:
         return repositories.keys()
 
+
 def backup_repository_info(name, human_readable=True, space_used=False):
     """
     Show info about a repository
@@ -189,7 +191,7 @@ def backup_repository_info(name, human_readable=True, space_used=False):
 
 @is_unit_operation()
 def backup_repository_add(operation_logger, location, name, description=None,
-                        methods=None, quota=None, encryption="passphrase"):
+                          methods=None, quota=None, encryption="passphrase"):
     """
     Add a backup repository
 

From 52ba72781967b196c938c645fde622799e7dd91b Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 29 Sep 2021 19:29:14 +0200
Subject: [PATCH 21/48] [wip] Read info json in backup method

---
 src/yunohost/backup.py | 92 +++++++++++++++++++++++++++++++-----------
 1 file changed, 69 insertions(+), 23 deletions(-)

diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index b02b23966..b1712ee96 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -1733,6 +1733,18 @@ class BackupMethod(object):
             logger.debug("unable to load info json", exc_info=1)
             raise YunohostError('backup_invalid_archive')
 
+	# (legacy) Retrieve backup size
+        # FIXME
+	size = info.get("size", 0)
+	if not size:
+            tar = tarfile.open(
+		archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
+            )
+            size = reduce(
+		lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
+            )
+            tar.close()
+
         return info
 
     def clean(self):
@@ -2137,6 +2149,7 @@ class TarBackupMethod(BackupMethod):
 
         return [remove_extension(f) for f in archives]
 
+
     def _archive_exists(self):
         return os.path.lexists(self.archive_path)
 
@@ -2154,21 +2167,63 @@ class TarBackupMethod(BackupMethod):
                                     path=archive_file)
 
     def _get_info_string(self):
-        info_file = os.path.join(self.repo.path, self.name + '.info.json')
+	name = self.name
+	if name.endswith(".tar.gz"):
+            name = name[: -len(".tar.gz")]
+	elif name.endswith(".tar"):
+            name = name[: -len(".tar")]
+
+	archive_file = "%s/%s.tar" % (self.repo.path, name)
+
+	# Check file exist (even if it's a broken symlink)
+	if not os.path.lexists(archive_file):
+            archive_file += ".gz"
+            if not os.path.lexists(archive_file):
+		raise YunohostValidationError("backup_archive_name_unknown", name=name)
+
+	# If symlink, retrieve the real path
+	if os.path.islink(archive_file):
+            archive_file = os.path.realpath(archive_file)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+		raise YunohostValidationError(
+                    "backup_archive_broken_link", path=archive_file
+		)
+
+	info_file = "%s/%s.info.json" % (self.repo.path, name)
+
+	if not os.path.exists(info_file):
+            tar = tarfile.open(
+		archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
+            )
+            info_dir = info_file + ".d"
 
-        if not os.path.exists(info_file):
-            tar = tarfile.open(self.archive_path, "r:gz")
-            info_dir = info_file + '.d'
             try:
-                tar.extract('info.json', path=info_dir)
+		files_in_archive = tar.getnames()
+            except (IOError, EOFError, tarfile.ReadError) as e:
+		raise YunohostError(
+                    "backup_archive_corrupted", archive=archive_file, error=str(e)
+		)
+
+            try:
+		if "info.json" in files_in_archive:
+                    tar.extract("info.json", path=info_dir)
+		elif "./info.json" in files_in_archive:
+                    tar.extract("./info.json", path=info_dir)
+		else:
+                    raise KeyError
             except KeyError:
-                logger.debug("unable to retrieve '%s' inside the archive",
-                             info_file, exc_info=1)
-                raise YunohostError('backup_invalid_archive')
+		logger.debug(
+                    "unable to retrieve '%s' inside the archive", info_file, exc_info=1
+		)
+		raise YunohostError(
+                    "backup_archive_cant_retrieve_info_json", archive=archive_file
+		)
             else:
-                shutil.move(os.path.join(info_dir, 'info.json'), info_file)
+		shutil.move(os.path.join(info_dir, "info.json"), info_file)
             finally:
-                tar.close()
+		tar.close()
             os.rmdir(info_dir)
 
         try:
@@ -2176,7 +2231,6 @@ class TarBackupMethod(BackupMethod):
         except MoulinetteError:
             logger.debug("unable to load '%s'", info_file, exc_info=1)
             raise YunohostError('backup_invalid_archive')
-        # FIXME : Don't we want to close the tar archive here or at some point ?
 
 
 class BorgBackupMethod(BackupMethod):
@@ -2661,19 +2715,11 @@ def backup_info(name, repo=None, with_details=False, human_readable=False):
 
     info = repo.method.info(name)
 
-    # Historically backup size was not here, in that case we know it's a tar archive
-    size = info.get('size', 0)
-    if not size:
-        tar = tarfile.open(repo.archive_path, "r:gz" if archive_file.endswith(".gz") else "r")
-        size = reduce(lambda x, y: getattr(x, 'size', x) + getattr(y, 'size', y),
-                      tar.getmembers())
-        tar.close()
-
     result = {
-        'path': repo.archive_path,
-        'created_at': datetime.utcfromtimestamp(info['created_at']),
-        'description': info['description'],
-        'size': size,
+        "path": repo.archive_path,
+        "created_at": datetime.utcfromtimestamp(info["created_at"]),
+        "description": info["description"],
+        "size": size,
     }
     if human_readable:
         result['size'] = binary_to_human(result['size']) + 'B'

From f0bff8f0211ffbb9ab4e8e77c787be13ee214851 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 1 Oct 2021 18:03:43 +0200
Subject: [PATCH 22/48] [wip] Rework backup repository with config panel

---
 data/actionsmap/yunohost.yml      |  46 ++++---
 data/other/config_repository.toml |  83 ++++++++++++
 src/yunohost/backup.py            |   4 +-
 src/yunohost/repository.py        | 211 +++++++++++++++++-------------
 4 files changed, 230 insertions(+), 114 deletions(-)
 create mode 100644 data/other/config_repository.toml

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 4adf3a07c..31790b83f 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -1161,33 +1161,43 @@ backup:
                 ### backup_repository_add()
                 add:
                     action_help: Add a backup repository
-                    api: POST /backup/repository/<name>
+                    api: POST /backup/repository/<shortname>
                     arguments:
-                        location:
+                        shortname:
+                            help: ID of the repository
+                            extra:
+                                pattern: &pattern_backup_repository_id
+                                    - !!str ^\w+$
+                                    - "pattern_backup_repository_id"
+                        -n:
+                            full: --name
+                            help: Short description of the repository
+                        -l:
+                            full: --location
                             help: Location on this server or on an other
                             extra:
                                 pattern: &pattern_backup_repository_location
                                     - !!str ^((ssh://)?[a-z_]\w*@\[\w\-\.]+:)?(~?/)?[\w/]*$
                                     - "pattern_backup_repository_location"
-                        -n:
-                            full: --name
-                            help: Name of the repository
-                            extra:
-                                pattern: &pattern_backup_repository_name
-                                    - !!str ^\w+$
-                                    - "pattern_backup_repository_name"
-                        -d:
-                            full: --description
-                            help: Short description of the repository
-                        --methods:
-                            help: List of backup methods accepted
-                            nargs: "*"
+                        -m:
+                            full: --method
+                            help: By default 'borg' method is used, could be 'tar' or a custom method
+                            default: borg
                         -q:
                             full: --quota
                             help: Quota to configure with this repository
-                        -e:
-                            full: --encryption
-                            help: Type of encryption
+                        -p:
+                            full: --passphrase
+                            help: A strong passphrase to encrypt/decrypt your backup (keep it preciously)
+                            action: store_true
+                        -a:
+                            full: --alert
+                            help: List of mails to which sent inactivity alert
+                            nargs: "*"
+                        -d:
+                            full: --alert-delay
+                            help: Inactivity delay in days after which we sent alerts mails
+                            default: 7
 
                 ### backup_repository_update()
                 update:
diff --git a/data/other/config_repository.toml b/data/other/config_repository.toml
new file mode 100644
index 000000000..b7be26d07
--- /dev/null
+++ b/data/other/config_repository.toml
@@ -0,0 +1,83 @@
+
+version = "1.0"
+i18n = "repository_config"
+[main]
+name.en = ""
+    []
+    name.en = ""
+        # if method == "tar": question["value"] = False
+        [creation] # TODO "Remote repository"
+        type = "boolean"
+        visible = "false"
+
+        [name] # TODO "Remote repository"
+        type = "string"
+
+        [is_remote] # TODO "Remote repository"
+        type = "boolean"
+        yes = true
+        no = false
+        visible = "creation && is_remote"
+
+        [is_f2f] # TODO "It's a YunoHost",
+        help = "" # "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
+        type = "boolean"
+        yes = true
+        no = false
+        visible = "creation && is_remote"
+
+        [public_key] # TODO "Here is the public key to give to your BorgBackup  provider : {public_key}"
+        type = "alert"
+        style = "info"
+        visible = "creation && is_remote && ! is_f2f"
+
+        [location]
+        ask = "Remote server domain"
+        type = "string"
+        visible = "creation && is_remote"
+        pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
+        pattern.error = '' # TODO "Please provide a valid domain"
+        # FIXME: can't be a domain of this instances ?
+
+        [alert] # TODO "Alert emails"
+        help = '' # TODO Declare emails to which sent inactivity alerts",
+        type = "tags"
+        visible = "is_remote && is_f2f"
+        pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
+        pattern.error = "It seems it's not a valid email"
+        # "value": alert,
+
+        [alert_delay] # TODO "Alert delay"
+        help = '' # TODO "After how many inactivity days send email alerts",
+        type = "number"
+        visible = "is_remote && is_f2f"
+        min = 1
+
+        [quota] # TODO "Quota"
+        type = "string"
+        visible = "is_remote && is_f2f"
+        pattern.regexp = '^\d+[MGT]$'
+        pattern.error = '' # TODO ""
+
+        [port] # TODO "Port"
+        type = "number"
+        visible = "is_remote && !is_f2f"
+        min = 1
+        max = 65535
+
+        [user] # TODO User
+        type = "string"
+        visible = "is_remote && !is_f2f"
+
+        [method] # TODO "Backup method"
+        type = "select"
+        # "value": method,
+        choices.borg = "BorgBackup (recommended)"
+        choices.tar = "Legacy tar archive mechanism"
+        default = "borg"
+        visible = "!is_remote"
+        
+        [path] # TODO "Archive path"
+        type = "path"
+        visible = "!is_remote or (is_remote and !is_f2f)"
+
diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index b1712ee96..a1794e9db 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -1647,7 +1647,7 @@ class BackupMethod(object):
         """
         self.manager = manager
         if not repo or isinstance(repo, basestring):
-            repo = BackupRepository.get_or_create(ARCHIVES_PATH)
+            repo = BackupRepository.get(ARCHIVES_PATH)
         self.repo = repo
 
     @property
@@ -2635,7 +2635,7 @@ def backup_list(repos=[], with_info=False, human_readable=False):
     result = OrderedDict()
 
     if repos == []:
-        repos = BackupRepository.all()
+        repos = backup_repository_list(full=True)
     else:
         for k, repo in repos:
             repos[k] = BackupRepository.get(repo)
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index cedafa8ae..446afdf65 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -27,6 +27,8 @@ import os
 import re
 import time
 import subprocess
+import re
+import urllib.parse
 
 from moulinette import msignals, m18n
 from moulinette.core import MoulinetteError
@@ -40,58 +42,43 @@ from yunohost.monitor import binary_to_human
 from yunohost.log import OperationLogger, is_unit_operation
 
 logger = getActionLogger('yunohost.repository')
-REPOSITORIES_PATH = '/etc/yunohost/repositories.yml'
+REPOSITORIES_PATH = '/etc/yunohost/repositories'
+REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
 
+# TODO
+# TODO i18n
+# TODO visible in cli
+# TODO split COnfigPanel.get to extract "Format result" part and be able to override it
+# TODO Migration
+# TODO Remove BackupRepository.get_or_create()
+# TODO Backup method
+# TODO auto test F2F by testing .well-known url
+# TODO API params to get description of forms
+# TODO tests
+# TODO detect external hard drive already mounted and suggest it
+# TODO F2F client detection / add / update / delete
+# TODO F2F server
 
-class BackupRepository(object):
+class BackupRepository(ConfigPanel):
     """
     BackupRepository manage all repository the admin added to the instance
     """
-    repositories = {}
-
     @classmethod
-    def create(cls, location, name, *args, **kwargs):
-        cls.load()
-
-        return BackupRepository(True, location, name, *args, **kwargs)
-
-    @classmethod
-    def get(cls, name):
-        cls.load()
-
+    def get(cls, shortname):
+        # FIXME
         if name not in cls.repositories:
             raise YunohostError('backup_repository_doesnt_exists', name=name)
 
-        return BackupRepository(False, **cls.repositories[name])
+        return cls.repositories[name]
 
-    @classmethod
-    def load(cls):
-        """
-        Read repositories configuration from file
-        """
-        if cls.repositories != {}:
-            return cls.repositories
+    def __init__(self, repository):
+        self.repository = repository
+        self.save_mode = "full"
+        super().__init__(
+            config_path=REPOSITORY_CONFIG_PATH,
+            save_path=f"{REPOSITORY_SETTINGS_DIR}/{repository}.yml",
+        )
 
-        if os.path.exists(REPOSITORIES_PATH):
-            try:
-                cls.repositories = read_yaml(REPOSITORIES_PATH)['repositories']
-            except MoulinetteError as e:
-                raise YunohostError(
-                    'backup_cant_open_repositories_file', reason=e)
-        return cls.repositories
-
-    @classmethod
-    def save(cls):
-        """
-        Save managed repositories to file
-        """
-        try:
-            write_to_json(REPOSITORIES_PATH, cls.repositories)
-        except Exception as e:
-            raise YunohostError('backup_cant_save_repositories_file', reason=e)
-
-    def __init__(self, created, location, name=None, description=None, method=None,
-                 encryption=None, quota=None):
 
         from yunohost.backup import BackupMethod
         self.location = location
@@ -102,12 +89,6 @@ class BackupRepository(object):
             raise YunohostError(
                 'backup_repository_already_exists', repositories=self.name)
 
-        self.description = description
-        self.encryption = encryption
-        self.quota = quota
-
-        if method is None:
-            method = 'tar' if self.domain is None else 'borg'
         self.method = BackupMethod.get(method, self)
 
     def list(self, with_info=False):
@@ -122,21 +103,16 @@ class BackupRepository(object):
         return self.used
 
     def purge(self):
+        # TODO F2F delete
         self.method.purge()
 
     def delete(self, purge=False):
-        repositories = BackupRepository.repositories
-
-        repositories.pop(self.name)
-
-        BackupRepository.save()
 
         if purge:
             self.purge()
 
-    def save(self):
-        BackupRepository.reposirories[self.name] = self.__dict__
-        BackupRepository.save()
+        os.system("rm -rf {REPOSITORY_SETTINGS_DIR}/{self.repository}.yml")
+
 
     def _split_location(self):
         """
@@ -159,27 +135,41 @@ def backup_repository_list(full=False):
     """
     List available repositories where put archives
     """
-    repositories = BackupRepository.load()
 
-    if full:
+    try:
+        repositories = [f.rstrip(".yml")
+                        for f in os.listdir(REPOSITORIES_PATH)
+                        if os.path.isfile(f) and f.endswith(".yml")]
+    except FileNotFoundError:
+        repositories = []
+
+    if not full:
         return repositories
-    else:
-        return repositories.keys()
+
+    # FIXME: what if one repo.yml is corrupted ?
+    repositories = {repo: BackupRepository(repo).get(mode="export")
+                    for repo in repositories}
+
+    return repositories
 
 
-def backup_repository_info(name, human_readable=True, space_used=False):
+def backup_repository_info(shortname, human_readable=True, space_used=False):
     """
     Show info about a repository
 
     Keyword arguments:
         name -- Name of the backup repository
     """
-    repository = BackupRepository.get(name)
-
+    Question.operation_logger = operation_logger
+    repository = BackupRepository(shortname)
+    # TODO
     if space_used:
         repository.compute_space_used()
 
-    repository = repository.__dict__
+    repository = repository.get(
+        mode="export"
+    )
+
     if human_readable:
         if 'quota' in repository:
             repository['quota'] = binary_to_human(repository['quota'])
@@ -190,58 +180,92 @@ def backup_repository_info(name, human_readable=True, space_used=False):
 
 
 @is_unit_operation()
-def backup_repository_add(operation_logger, location, name, description=None,
-                          methods=None, quota=None, encryption="passphrase"):
+def backup_repository_add(operation_logger, shortname, name=None, location=None,
+                          method=None, quota=None, passphrase=None,
+                          alert=[], alert_delay=7):
     """
     Add a backup repository
 
     Keyword arguments:
         location -- Location of the repository (could be a remote location)
-        name -- Name of the backup repository
-        description -- An optionnal description
+        shortname -- Name of the backup repository
+        name -- An optionnal description
         quota -- Maximum size quota of the repository
         encryption -- If available, the kind of encryption to use
     """
-    repository = BackupRepository(
-        location, name, description, methods, quota, encryption)
+    # FIXME i18n
+    # Deduce some value from location
+    args = {}
+    args['name'] = name
+    args['creation'] = True
+    if location:
+        args["location"] = location
+        args["is_remote"] = True
+        args["method"] = method if method else "borg"
+        domain_re = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
+        if re.match(domain_re, location):
+            args["is_f2f"] = True
+        elif location[0] != "/":
+            args["is_f2f"] = False
+        else:
+            args["is_remote"] = False
+            args["method"] = method
+    elif method == "tar":
+        args["is_remote"] = False
+    if not location:
+        args["method"] = method
 
-    try:
-        repository.save()
-    except MoulinetteError:
-        raise YunohostError('backup_repository_add_failed',
-                            repository=name, location=location)
+    args["quota"] = quota
+    args["passphrase"] = passphrase
+    args["alert"]= ",".join(alert) if alert else None
+    args["alert_delay"]= alert_delay
 
-    logger.success(m18n.n('backup_repository_added',
-                        repository=name, location=location))
+    # TODO validation
+    # TODO activate service in apply (F2F or not)
+    Question.operation_logger = operation_logger
+    repository = BackupRepository(shortname)
+    return repository.set(
+        args=urllib.parse.urlencode(args),
+        operation_logger=operation_logger
+    )
 
 
 @is_unit_operation()
-def backup_repository_update(operation_logger, name, description=None,
-                            quota=None, password=None):
+def backup_repository_update(operation_logger, shortname, name=None,
+                             quota=None, passphrase=None,
+                             alert=[], alert_delay=None):
     """
     Update a backup repository
 
     Keyword arguments:
         name -- Name of the backup repository
     """
-    repository = BackupRepository.get(name)
 
-    if description is not None:
-        repository.description = description
+    args = {}
+    args['creation'] = False
+    if name:
+        args['name'] = name
+    if quota:
+        args["quota"] = quota
+    if passphrase:
+        args["passphrase"] = passphrase
+    if alert is not None:
+        args["alert"]= ",".join(alert) if alert else None
+    if alert_delay:
+        args["alert_delay"]= alert_delay
 
-    if quota is not None:
-        repository.quota = quota
-
-    try:
-        repository.save()
-    except MoulinetteError:
-        raise YunohostError('backup_repository_update_failed', repository=name)
-    logger.success(m18n.n('backup_repository_updated', repository=name,
-                        location=repository['location']))
+    # TODO validation
+    # TODO activate service in apply
+    Question.operation_logger = operation_logger
+    repository = BackupRepository(shortname)
+    return repository.set(
+        args=urllib.parse.urlencode(args),
+        operation_logger=operation_logger
+    )
 
 
 @is_unit_operation()
-def backup_repository_remove(operation_logger, name, purge=False):
+def backup_repository_remove(operation_logger, shortname, purge=False):
     """
     Remove a backup repository
 
@@ -249,7 +273,6 @@ def backup_repository_remove(operation_logger, name, purge=False):
         name -- Name of the backup repository to remove
 
     """
-    repository = BackupRepository.get(name)
-    repository.delete(purge)
-    logger.success(m18n.n('backup_repository_removed', repository=name,
+    BackupRepository(shortname).delete(purge)
+    logger.success(m18n.n('backup_repository_removed', repository=shortname,
                         path=repository['path']))

From 8376cade21f66d6feae192d4dc5456b6d59eefc9 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 6 Oct 2021 18:25:57 +0200
Subject: [PATCH 23/48] [wip] Repository add

---
 data/actionsmap/yunohost.yml      |  10 +--
 data/other/config_repository.toml |  67 ++++++++++--------
 locales/en.json                   |  15 ++++
 src/yunohost/backup.py            | 112 ++++++++++++------------------
 src/yunohost/repository.py        |  33 ++++-----
 src/yunohost/utils/config.py      |  13 ++--
 src/yunohost/utils/filesystem.py  |  27 +++++++
 src/yunohost/utils/i18n.py        |   4 +-
 src/yunohost/utils/network.py     |  11 +++
 9 files changed, 165 insertions(+), 127 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 31790b83f..99ce3158f 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -1166,9 +1166,9 @@ backup:
                         shortname:
                             help: ID of the repository
                             extra:
-                                pattern: &pattern_backup_repository_id
-                                    - !!str ^\w+$
-                                    - "pattern_backup_repository_id"
+                                pattern: &pattern_backup_repository_shortname
+                                    - !!str ^[a-zA-Z0-9-_]+$
+                                    - "pattern_backup_repository_shortname"
                         -n:
                             full: --name
                             help: Short description of the repository
@@ -1207,7 +1207,7 @@ backup:
                         name:
                             help: Name of the backup repository to update
                             extra:
-                                pattern: *pattern_backup_repository_name
+                                pattern: *pattern_backup_repository_shortname
                         -d:
                             full: --description
                             help: Short description of the repository
@@ -1229,7 +1229,7 @@ backup:
                         name:
                             help: Name of the backup repository to remove
                             extra:
-                                pattern: *pattern_backup_repository_name
+                                pattern: *pattern_backup_repository_shortname
                         --purge:
                             help: Remove all archives and data inside repository
                             action: store_false
diff --git a/data/other/config_repository.toml b/data/other/config_repository.toml
index b7be26d07..748f7f68a 100644
--- a/data/other/config_repository.toml
+++ b/data/other/config_repository.toml
@@ -3,73 +3,79 @@ version = "1.0"
 i18n = "repository_config"
 [main]
 name.en = ""
-    []
+    [main.main]
     name.en = ""
         # if method == "tar": question["value"] = False
-        [creation] # TODO "Remote repository"
-        type = "boolean"
-        visible = "false"
-
-        [name] # TODO "Remote repository"
+        [main.main.description]
         type = "string"
+        default = ""
 
-        [is_remote] # TODO "Remote repository"
+        [main.main.is_remote]
         type = "boolean"
         yes = true
         no = false
         visible = "creation && is_remote"
+        default = "no"
 
-        [is_f2f] # TODO "It's a YunoHost",
-        help = "" # "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
-        type = "boolean"
-        yes = true
-        no = false
-        visible = "creation && is_remote"
-
-        [public_key] # TODO "Here is the public key to give to your BorgBackup  provider : {public_key}"
-        type = "alert"
-        style = "info"
-        visible = "creation && is_remote && ! is_f2f"
-
-        [location]
-        ask = "Remote server domain"
+        [main.main.location]
+        ask.en = "{is_remote}"
         type = "string"
         visible = "creation && is_remote"
         pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
         pattern.error = '' # TODO "Please provide a valid domain"
+        default = ""
         # FIXME: can't be a domain of this instances ?
 
-        [alert] # TODO "Alert emails"
-        help = '' # TODO Declare emails to which sent inactivity alerts",
+        [main.main.is_f2f]
+        ask.en = "{is_remote}"
+        help = ""
+        type = "boolean"
+        yes = true
+        no = false
+        visible = "creation && is_remote"
+        default = "no"
+
+        [main.main.public_key]
+        type = "alert"
+        style = "info"
+        visible = "creation && is_remote && ! is_f2f"
+
+        [main.main.alert]
+        help = ''
         type = "tags"
         visible = "is_remote && is_f2f"
         pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
         pattern.error = "It seems it's not a valid email"
+        default = []
         # "value": alert,
 
-        [alert_delay] # TODO "Alert delay"
-        help = '' # TODO "After how many inactivity days send email alerts",
+        [main.main.alert_delay]
+        help = ''
         type = "number"
         visible = "is_remote && is_f2f"
         min = 1
+        default = 7
 
-        [quota] # TODO "Quota"
+        [main.main.quota]
         type = "string"
         visible = "is_remote && is_f2f"
         pattern.regexp = '^\d+[MGT]$'
         pattern.error = '' # TODO ""
+        default = ""
 
-        [port] # TODO "Port"
+        [main.main.port]
         type = "number"
         visible = "is_remote && !is_f2f"
         min = 1
         max = 65535
+        default = 22
 
-        [user] # TODO User
+        [main.main.user]
         type = "string"
         visible = "is_remote && !is_f2f"
+        default = ""
 
-        [method] # TODO "Backup method"
+        [main.main.method]
         type = "select"
         # "value": method,
         choices.borg = "BorgBackup (recommended)"
@@ -77,7 +83,8 @@ name.en = ""
         default = "borg"
         visible = "!is_remote"
         
-        [path] # TODO "Archive path"
+        [main.main.path]
         type = "path"
         visible = "!is_remote or (is_remote and !is_f2f)"
+        default = "/home/yunohost.backup/archives"
 
diff --git a/locales/en.json b/locales/en.json
index cf24cfc09..49505764d 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -593,6 +593,21 @@
     "regenconf_would_be_updated": "The configuration would have been updated for category '{category}'",
     "regex_incompatible_with_tile": "/!\\ Packagers! Permission '{permission}' has show_tile set to 'true' and you therefore cannot define a regex URL as the main URL",
     "regex_with_only_domain": "You can't use a regex for domain, only for path",
+    "repository_config_description": "Long name",
+    "repository_config_is_remote": "Remote repository",
+    "repository_config_is_f2f": "It's a YunoHost",
+    "repository_config_is_f2f_help": "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
+    "repository_config_location": "Remote server domain",
+    "repository_config_public_key": "Public key to give to your BorgBackup  provider : {public_key}",
+    "repository_config_alert": "Alert emails",
+    "repository_config_alert_help": "Declare emails to which sent inactivity alerts",
+    "repository_config_alert_delay": "Alert delay",
+    "repository_config_alert_delay_help": "After how many inactivity days send email alerts",
+    "repository_config_quota": "Quota",
+    "repository_config_port": "Port",
+    "repository_config_user": "User",
+    "repository_config_method": "Method",
+    "repository_config_path": "Archive path",
     "restore_already_installed_app": "An app with the ID '{app}' is already installed",
     "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}",
     "restore_backup_too_old": "This backup archive can not be restored because it comes from a too-old YunoHost version.",
diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index a1794e9db..3174f59f9 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -73,7 +73,7 @@ from yunohost.log import OperationLogger, is_unit_operation
 from yunohost.repository import BackupRepository
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.packages import ynh_packages_version
-from yunohost.utils.filesystem import free_space_in_directory
+from yunohost.utils.filesystem import free_space_in_directory, disk_usage, binary_to_human
 from yunohost.settings import settings_get
 
 BACKUP_PATH = "/home/yunohost.backup"
@@ -1733,15 +1733,15 @@ class BackupMethod(object):
             logger.debug("unable to load info json", exc_info=1)
             raise YunohostError('backup_invalid_archive')
 
-	# (legacy) Retrieve backup size
+        # (legacy) Retrieve backup size
         # FIXME
-	size = info.get("size", 0)
-	if not size:
+        size = info.get("size", 0)
+        if not size:
             tar = tarfile.open(
-		archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
+                archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
             )
             size = reduce(
-		lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
+                lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
             )
             tar.close()
 
@@ -2167,63 +2167,63 @@ class TarBackupMethod(BackupMethod):
                                     path=archive_file)
 
     def _get_info_string(self):
-	name = self.name
-	if name.endswith(".tar.gz"):
+        name = self.name
+        if name.endswith(".tar.gz"):
             name = name[: -len(".tar.gz")]
-	elif name.endswith(".tar"):
+        elif name.endswith(".tar"):
             name = name[: -len(".tar")]
 
-	archive_file = "%s/%s.tar" % (self.repo.path, name)
+        archive_file = "%s/%s.tar" % (self.repo.path, name)
 
-	# Check file exist (even if it's a broken symlink)
-	if not os.path.lexists(archive_file):
+        # Check file exist (even if it's a broken symlink)
+        if not os.path.lexists(archive_file):
             archive_file += ".gz"
             if not os.path.lexists(archive_file):
-		raise YunohostValidationError("backup_archive_name_unknown", name=name)
+                raise YunohostValidationError("backup_archive_name_unknown", name=name)
 
-	# If symlink, retrieve the real path
-	if os.path.islink(archive_file):
+        # If symlink, retrieve the real path
+        if os.path.islink(archive_file):
             archive_file = os.path.realpath(archive_file)
 
             # Raise exception if link is broken (e.g. on unmounted external storage)
             if not os.path.exists(archive_file):
-		raise YunohostValidationError(
+                raise YunohostValidationError(
                     "backup_archive_broken_link", path=archive_file
-		)
+                )
 
-	info_file = "%s/%s.info.json" % (self.repo.path, name)
+        info_file = "%s/%s.info.json" % (self.repo.path, name)
 
-	if not os.path.exists(info_file):
+        if not os.path.exists(info_file):
             tar = tarfile.open(
-		archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
+                archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
             )
             info_dir = info_file + ".d"
 
             try:
-		files_in_archive = tar.getnames()
+                files_in_archive = tar.getnames()
             except (IOError, EOFError, tarfile.ReadError) as e:
-		raise YunohostError(
+                raise YunohostError(
                     "backup_archive_corrupted", archive=archive_file, error=str(e)
-		)
+                )
 
             try:
-		if "info.json" in files_in_archive:
+                if "info.json" in files_in_archive:
                     tar.extract("info.json", path=info_dir)
-		elif "./info.json" in files_in_archive:
+                elif "./info.json" in files_in_archive:
                     tar.extract("./info.json", path=info_dir)
-		else:
+                else:
                     raise KeyError
             except KeyError:
-		logger.debug(
+                logger.debug(
                     "unable to retrieve '%s' inside the archive", info_file, exc_info=1
-		)
-		raise YunohostError(
+                )
+                raise YunohostError(
                     "backup_archive_cant_retrieve_info_json", archive=archive_file
-		)
+                )
             else:
-		shutil.move(os.path.join(info_dir, "info.json"), info_file)
+                shutil.move(os.path.join(info_dir, "info.json"), info_file)
             finally:
-		tar.close()
+                tar.close()
             os.rmdir(info_dir)
 
         try:
@@ -2804,24 +2804,28 @@ def backup_delete(name):
 import yunohost.repository
 
 
-def backup_repository_list(full):
+def backup_repository_list(full=False):
     return yunohost.repository.backup_repository_list(full)
 
 
-def backup_repository_info(name, human_readable, space_used):
-    return yunohost.repository.backup_repository_info(name, human_readable, space_used)
+def backup_repository_info(shortname, human_readable=True, space_used=False):
+    return yunohost.repository.backup_repository_info(shortname, human_readable, space_used)
 
 
-def backup_repository_add(location, name, description, methods, quota, encryption):
-    return yunohost.repository.backup_repository_add(location, name, description, methods, quota, encryption)
+def backup_repository_add(shortname, name=None, location=None,
+                          method=None, quota=None, passphrase=None,
+                          alert=[], alert_delay=7):
+    return yunohost.repository.backup_repository_add(location, shortname, name, method, quota, passphrase, alert, alert_delay)
 
 
-def backup_repository_update(name, description, quota, password):
-    return yunohost.repository.backup_repository_update(name, description, quota, password)
+def backup_repository_update(shortname, name=None,
+                             quota=None, passphrase=None,
+                             alert=[], alert_delay=None):
+    return yunohost.repository.backup_repository_update(shortname, name, quota, passphrase, alert, alert_delay)
 
 
-def backup_repository_remove(name, purge):
-    return yunohost.repository.backup_repository_remove(name, purge)
+def backup_repository_remove(shortname, purge=False):
+    return yunohost.repository.backup_repository_remove(shortname, purge)
 
 
 
@@ -2882,29 +2886,3 @@ def _recursive_umount(directory):
     return everything_went_fine
 
 
-def disk_usage(path):
-    # We don't do this in python with os.stat because we don't want
-    # to follow symlinks
-
-    du_output = check_output(["du", "-sb", path], shell=False)
-    return int(du_output.split()[0])
-
-
-def binary_to_human(n, customary=False):
-    """
-    Convert bytes or bits into human readable format with binary prefix
-    Keyword argument:
-        n -- Number to convert
-        customary -- Use customary symbol instead of IEC standard
-    """
-    symbols = ("Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
-    if customary:
-        symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
-    prefix = {}
-    for i, s in enumerate(symbols):
-        prefix[s] = 1 << (i + 1) * 10
-    for s in reversed(symbols):
-        if n >= prefix[s]:
-            value = float(n) / prefix[s]
-            return "%.1f%s" % (value, s)
-    return "%s" % n
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 446afdf65..5f06c5102 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -30,24 +30,24 @@ import subprocess
 import re
 import urllib.parse
 
-from moulinette import msignals, m18n
+from moulinette import Moulinette, m18n
 from moulinette.core import MoulinetteError
 from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file, read_yaml, write_to_json
 
 
+from yunohost.utils.config import ConfigPanel, Question
 from yunohost.utils.error import YunohostError
-from yunohost.monitor import binary_to_human
+from yunohost.utils.filesystem import binary_to_human
+from yunohost.utils.network import get_ssh_public_key
 from yunohost.log import OperationLogger, is_unit_operation
 
 logger = getActionLogger('yunohost.repository')
-REPOSITORIES_PATH = '/etc/yunohost/repositories'
+REPOSITORIES_DIR = '/etc/yunohost/repositories'
 REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
 
-# TODO
 # TODO i18n
-# TODO visible in cli
 # TODO split COnfigPanel.get to extract "Format result" part and be able to override it
 # TODO Migration
 # TODO Remove BackupRepository.get_or_create()
@@ -76,20 +76,15 @@ class BackupRepository(ConfigPanel):
         self.save_mode = "full"
         super().__init__(
             config_path=REPOSITORY_CONFIG_PATH,
-            save_path=f"{REPOSITORY_SETTINGS_DIR}/{repository}.yml",
+            save_path=f"{REPOSITORIES_DIR}/{repository}.yml",
         )
 
+        #self.method = BackupMethod.get(method, self)
 
-        from yunohost.backup import BackupMethod
-        self.location = location
-        self._split_location()
-
-        self.name = location if name is None else name
-        if created and self.name in BackupMethod.repositories:
-            raise YunohostError(
-                'backup_repository_already_exists', repositories=self.name)
-
-        self.method = BackupMethod.get(method, self)
+    def _get_default_values(self):
+        values = super()._get_default_values()
+        values["public_key"] = get_ssh_public_key()
+        return values
 
     def list(self, with_info=False):
         return self.method.list(with_info)
@@ -138,7 +133,7 @@ def backup_repository_list(full=False):
 
     try:
         repositories = [f.rstrip(".yml")
-                        for f in os.listdir(REPOSITORIES_PATH)
+                        for f in os.listdir(REPOSITORIES_DIR)
                         if os.path.isfile(f) and f.endswith(".yml")]
     except FileNotFoundError:
         repositories = []
@@ -196,7 +191,7 @@ def backup_repository_add(operation_logger, shortname, name=None, location=None,
     # FIXME i18n
     # Deduce some value from location
     args = {}
-    args['name'] = name
+    args['description'] = name
     args['creation'] = True
     if location:
         args["location"] = location
@@ -244,7 +239,7 @@ def backup_repository_update(operation_logger, shortname, name=None,
     args = {}
     args['creation'] = False
     if name:
-        args['name'] = name
+        args['description'] = name
     if quota:
         args["quota"] = quota
     if passphrase:
diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py
index 27a9e1533..add5463f9 100644
--- a/src/yunohost/utils/config.py
+++ b/src/yunohost/utils/config.py
@@ -86,7 +86,7 @@ class ConfigPanel:
             if "ask" in option:
                 ask = _value_for_locale(option["ask"])
             elif "i18n" in self.config:
-                ask = m18n.n(self.config["i18n"] + "_" + option["id"])
+                ask = m18n.n(self.config["i18n"] + "_" + option["id"], **self.values)
 
             if mode == "full":
                 # edit self.config directly
@@ -298,7 +298,9 @@ class ConfigPanel:
                         logger.warning(f"Unknown key '{key}' found in config panel")
                     # Todo search all i18n keys
                     out[key] = (
-                        value if key not in ["ask", "help", "name"] else {"en": value}
+                        value
+                        if key not in ["ask", "help", "name"] or isinstance(value, (dict, OrderedDict))
+                        else {"en": value}
                     )
             return out
 
@@ -367,7 +369,7 @@ class ConfigPanel:
         if "i18n" in self.config:
             for panel, section, option in self._iterate():
                 if "ask" not in option:
-                    option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"])
+                    option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"], **self.values)
 
         def display_header(message):
             """CLI panel/section header display"""
@@ -377,7 +379,8 @@ class ConfigPanel:
         for panel, section, obj in self._iterate(["panel", "section"]):
             if panel == obj:
                 name = _value_for_locale(panel["name"])
-                display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
+                if name:
+                    display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
                 continue
             name = _value_for_locale(section["name"])
             if name:
@@ -570,7 +573,7 @@ class Question(object):
 
     def _format_text_for_user_input_in_cli(self):
 
-        text_for_user_input_in_cli = _value_for_locale(self.ask)
+        text_for_user_input_in_cli = _value_for_locale(self.ask).format(**self.context)
 
         if self.choices:
 
diff --git a/src/yunohost/utils/filesystem.py b/src/yunohost/utils/filesystem.py
index 04d7d3906..494a0187c 100644
--- a/src/yunohost/utils/filesystem.py
+++ b/src/yunohost/utils/filesystem.py
@@ -29,3 +29,30 @@ def free_space_in_directory(dirpath):
 def space_used_by_directory(dirpath):
     stat = os.statvfs(dirpath)
     return stat.f_frsize * stat.f_blocks
+
+def disk_usage(path):
+    # We don't do this in python with os.stat because we don't want
+    # to follow symlinks
+
+    du_output = check_output(["du", "-sb", path], shell=False)
+    return int(du_output.split()[0])
+
+
+def binary_to_human(n, customary=False):
+    """
+    Convert bytes or bits into human readable format with binary prefix
+    Keyword argument:
+        n -- Number to convert
+        customary -- Use customary symbol instead of IEC standard
+    """
+    symbols = ("Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
+    if customary:
+        symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
+    prefix = {}
+    for i, s in enumerate(symbols):
+        prefix[s] = 1 << (i + 1) * 10
+    for s in reversed(symbols):
+        if n >= prefix[s]:
+            value = float(n) / prefix[s]
+            return "%.1f%s" % (value, s)
+    return "%s" % n
diff --git a/src/yunohost/utils/i18n.py b/src/yunohost/utils/i18n.py
index a0daf8181..ab0b4253d 100644
--- a/src/yunohost/utils/i18n.py
+++ b/src/yunohost/utils/i18n.py
@@ -18,6 +18,8 @@
     along with this program; if not, see http://www.gnu.org/licenses
 
 """
+from collections import OrderedDict
+
 from moulinette import m18n
 
 
@@ -32,7 +34,7 @@ def _value_for_locale(values):
         An utf-8 encoded string
 
     """
-    if not isinstance(values, dict):
+    if not isinstance(values, (dict, OrderedDict)):
         return values
 
     for lang in [m18n.locale, m18n.default_locale]:
diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py
index 4474af14f..3d3045e6f 100644
--- a/src/yunohost/utils/network.py
+++ b/src/yunohost/utils/network.py
@@ -165,3 +165,14 @@ def _extract_inet(string, skip_netmask=False, skip_loopback=True):
         break
 
     return result
+
+def get_ssh_public_key():
+    keys = [
+        '/etc/ssh/ssh_host_ed25519_key.pub',
+        '/etc/ssh/ssh_host_rsa_key.pub'
+    ]
+    for key in keys:
+        if os.path.exists(key):
+            # We return the key without user and machine name.
+            # Providers don't need this info.
+            return " ".join(read_file(key).split(" ")[0:2])

From 27273c572119e8bec8b2bd9436ddaf903d3de14a Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 6 Oct 2021 19:11:25 +0200
Subject: [PATCH 24/48] [wip] Hook system in config panel

---
 data/other/config_repository.toml |  7 +++----
 src/yunohost/repository.py        |  4 ++++
 src/yunohost/utils/config.py      | 18 ++++++++++++++----
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/data/other/config_repository.toml b/data/other/config_repository.toml
index 748f7f68a..76c7fa987 100644
--- a/data/other/config_repository.toml
+++ b/data/other/config_repository.toml
@@ -5,6 +5,7 @@ i18n = "repository_config"
 name.en = ""
     [main.main]
     name.en = ""
+    optional = false
         # if method == "tar": question["value"] = False
         [main.main.description]
         type = "string"
@@ -14,20 +15,18 @@ name.en = ""
         type = "boolean"
         yes = true
         no = false
-        visible = "creation && is_remote"
+        visible = "creation"
         default = "no"
 
         [main.main.location]
-        ask.en = "{is_remote}"
         type = "string"
         visible = "creation && is_remote"
         pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        pattern.error = '' # TODO "Please provide a valid domain"
+        pattern.error = 'location_error' # TODO "Please provide a valid domain"
         default = ""
         # FIXME: can't be a domain of this instances ?
 
         [main.main.is_f2f]
-        ask.en = "{is_remote}"
         help = ""
         type = "boolean"
         yes = true
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 5f06c5102..ab5e6d62b 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -81,6 +81,10 @@ class BackupRepository(ConfigPanel):
 
         #self.method = BackupMethod.get(method, self)
 
+    def set__domain(self, question):
+        # TODO query on domain name .well-known
+        question.value
+
     def _get_default_values(self):
         values = super()._get_default_values()
         values["public_key"] = get_ssh_public_key()
diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py
index dedf13621..aafacbdd4 100644
--- a/src/yunohost/utils/config.py
+++ b/src/yunohost/utils/config.py
@@ -667,7 +667,7 @@ class Question(object):
             # - we want to keep the previous value
             # - we want the default value
             self.value = None
-            return self.value
+            return { self.name: self.value }
 
         for i in range(5):
             # Display question if no value filled or if it's a readonly message
@@ -689,6 +689,10 @@ class Question(object):
                 # Normalize and validate
                 self.value = self.normalize(self.value, self)
                 self._prevalidate()
+                # Search for validator in hooks
+                validator = f"validate__{self.name}"
+                if validator in self.hooks:
+                    self.hooks[validator](self)
             except YunohostValidationError as e:
                 # If in interactive cli, re-ask the current question
                 if i < 4 and Moulinette.interface.type == "cli" and os.isatty(1):
@@ -703,7 +707,12 @@ class Question(object):
 
         self.value = self._post_parse_value()
 
-        return self.value
+        # Search for post actions in hooks
+        post_hook = f"post_parse__{self.name}"
+        if post_hook in self.hooks:
+            self.hooks[post_hook](self)
+
+        return { self.name: self.value }
 
     def _prevalidate(self):
         if self.value in [None, ""] and not self.optional:
@@ -1217,7 +1226,8 @@ ARGUMENTS_TYPE_PARSERS = {
 
 
 def ask_questions_and_parse_answers(
-    raw_questions: Dict, prefilled_answers: Union[str, Mapping[str, Any]] = {}
+    raw_questions: Dict, prefilled_answers: Union[str, Mapping[str, Any]] = {},
+    hooks: Dict[str, Callable[[], None]] = {}
 ) -> List[Question]:
     """Parse arguments store in either manifest.json or actions.json or from a
     config panel against the user answers when they are present.
@@ -1251,7 +1261,7 @@ def ask_questions_and_parse_answers(
         question_class = ARGUMENTS_TYPE_PARSERS[raw_question.get("type", "string")]
         raw_question["value"] = answers.get(raw_question["name"])
         question = question_class(raw_question, context=answers)
-        answers[question.name] = question.ask_if_needed()
+        answers.update(question.ask_if_needed())
         out.append(question)
 
     return out

From c0cd8dbf074c2819c8ce9459a56246ed32e53478 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 13 Oct 2021 19:28:15 +0200
Subject: [PATCH 25/48] [enh] config panel python method hook

---
 src/yunohost/app.py          |  19 ++--
 src/yunohost/domain.py       |  11 +--
 src/yunohost/utils/config.py | 164 +++++++++++++++++++++++++----------
 3 files changed, 127 insertions(+), 67 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index fe5281384..c78b7a7f3 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1608,14 +1608,9 @@ def app_config_set(
 
 
 class AppConfigPanel(ConfigPanel):
-    def __init__(self, app):
-
-        # Check app is installed
-        _assert_is_installed(app)
-
-        self.app = app
-        config_path = os.path.join(APPS_SETTING_PATH, app, "config_panel.toml")
-        super().__init__(config_path=config_path)
+    entity_type = "app"
+    save_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/settings.yml")
+    config_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/config_panel.yml")
 
     def _load_current_values(self):
         self.values = self._call_config_script("show")
@@ -1639,7 +1634,7 @@ class AppConfigPanel(ConfigPanel):
         from yunohost.hook import hook_exec
 
         # Add default config script if needed
-        config_script = os.path.join(APPS_SETTING_PATH, self.app, "scripts", "config")
+        config_script = os.path.join(APPS_SETTING_PATH, self.entity, "scripts", "config")
         if not os.path.exists(config_script):
             logger.debug("Adding a default config script")
             default_script = """#!/bin/bash
@@ -1651,15 +1646,15 @@ ynh_app_config_run $1
 
         # Call config script to extract current values
         logger.debug(f"Calling '{action}' action from config script")
-        app_id, app_instance_nb = _parse_app_instance_name(self.app)
+        app_id, app_instance_nb = _parse_app_instance_name(self.entity)
         settings = _get_app_settings(app_id)
         env.update(
             {
                 "app_id": app_id,
-                "app": self.app,
+                "app": self.entity,
                 "app_instance_nb": str(app_instance_nb),
                 "final_path": settings.get("final_path", ""),
-                "YNH_APP_BASEDIR": os.path.join(APPS_SETTING_PATH, self.app),
+                "YNH_APP_BASEDIR": os.path.join(APPS_SETTING_PATH, self.entity),
             }
         )
 
diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py
index b40831d25..b353badb8 100644
--- a/src/yunohost/domain.py
+++ b/src/yunohost/domain.py
@@ -452,14 +452,9 @@ def domain_config_set(
 
 
 class DomainConfigPanel(ConfigPanel):
-    def __init__(self, domain):
-        _assert_domain_exists(domain)
-        self.domain = domain
-        self.save_mode = "diff"
-        super().__init__(
-            config_path=DOMAIN_CONFIG_PATH,
-            save_path=f"{DOMAIN_SETTINGS_DIR}/{domain}.yml",
-        )
+    entity_type = "domain"
+    save_path_tpl = f"{DOMAIN_SETTINGS_PATH}/{entity}.yml")
+    save_mode = "diff"
 
     def _get_toml(self):
         from yunohost.dns import _get_registrar_config_section
diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py
index aafacbdd4..aa1f2e7e8 100644
--- a/src/yunohost/utils/config.py
+++ b/src/yunohost/utils/config.py
@@ -19,6 +19,7 @@
 
 """
 
+import glob
 import os
 import re
 import urllib.parse
@@ -27,7 +28,7 @@ import shutil
 import ast
 import operator as op
 from collections import OrderedDict
-from typing import Optional, Dict, List, Union, Any, Mapping
+from typing import Optional, Dict, List, Union, Any, Mapping, Callable
 
 from moulinette.interfaces.cli import colorize
 from moulinette import Moulinette, m18n
@@ -189,13 +190,45 @@ def evaluate_simple_js_expression(expr, context={}):
 
 
 class ConfigPanel:
-    def __init__(self, config_path, save_path=None):
+    entity_type = "config"
+    save_path_tpl = None
+    config_path_tpl = "/usr/share/yunohost/other/config_{entity}.toml"
+
+    @classmethod
+    def list(cls):
+        """
+        List available config panel
+        """
+        try:
+            entities = [re.match("^" + cls.save_path_tpl.format(entity="(?p<entity>)") + "$", f).group('entity')
+                        for f in glob.glob(cls.save_path_tpl.format(entity="*"))
+                        if os.path.isfile(f)]
+        except FileNotFoundError:
+            entities = []
+        return entities
+
+    def __init__(self, entity, config_path=None, save_path=None, creation=False):
+        self.entity = entity
         self.config_path = config_path
+        if not config_path:
+            self.config_path = self.config_path_tpl.format(entity=entity)
         self.save_path = save_path
+        if not save_path and self.save_path_tpl:
+            self.save_path = self.save_path_tpl.format(entity=entity)
         self.config = {}
         self.values = {}
         self.new_values = {}
 
+        if self.save_path and not creation and not os.path.exists(self.save_path):
+            raise YunohostError(f"{self.entity_type}_doesnt_exists", name=entity)
+        if self.save_path and creation and os.path.exists(self.save_path):
+            raise YunohostError(f"{self.entity_type}_already_exists", name=entity)
+
+        # Search for hooks in the config panel
+        self.hooks = {func: getattr(self, func)
+                   for func in dir(self)
+                   if callable(getattr(self, func)) and re.match("^(validate|post_ask)__", func)}
+
     def get(self, key="", mode="classic"):
         self.filter_key = key or ""
 
@@ -274,19 +307,12 @@ class ConfigPanel:
 
         # Import and parse pre-answered options
         logger.debug("Import and parse pre-answered options")
-        args = urllib.parse.parse_qs(args or "", keep_blank_values=True)
-        self.args = {key: ",".join(value_) for key, value_ in args.items()}
-
-        if args_file:
-            # Import YAML / JSON file but keep --args values
-            self.args = {**read_yaml(args_file), **self.args}
-
-        if value is not None:
-            self.args = {self.filter_key.split(".")[-1]: value}
+        self._parse_pre_answered(args, value, args_file)
 
         # Read or get values and hydrate the config
         self._load_current_values()
         self._hydrate()
+        Question.operation_logger = operation_logger
         self._ask()
 
         if operation_logger:
@@ -528,7 +554,12 @@ class ConfigPanel:
                 display_header(f"\n# {name}")
 
             # Check and ask unanswered questions
-            questions = ask_questions_and_parse_answers(section["options"], self.args)
+            questions = ask_questions_and_parse_answers(
+                section["options"],
+                prefilled_answers=self.args,
+                current_values=self.values,
+                hooks=self.hooks
+            )
             self.new_values.update(
                 {
                     question.name: question.value
@@ -546,20 +577,42 @@ class ConfigPanel:
             if "default" in option
         }
 
+    @property
+    def future_values(self): # TODO put this in ConfigPanel ?
+        return {**self.values, **self.new_values}
+
+    def __getattr__(self, name):
+        if "new_values" in self.__dict__ and name in self.new_values:
+            return self.new_values[name]
+
+        if "values" in self.__dict__ and name in self.values:
+            return self.values[name]
+
+        return self.__dict__[name]
+
     def _load_current_values(self):
         """
         Retrieve entries in YAML file
         And set default values if needed
         """
 
-        # Retrieve entries in the YAML
-        on_disk_settings = {}
-        if os.path.exists(self.save_path) and os.path.isfile(self.save_path):
-            on_disk_settings = read_yaml(self.save_path) or {}
-
         # Inject defaults if needed (using the magic .update() ;))
         self.values = self._get_default_values()
-        self.values.update(on_disk_settings)
+
+        # Retrieve entries in the YAML
+        if os.path.exists(self.save_path) and os.path.isfile(self.save_path):
+            self.values.update(read_yaml(self.save_path) or {})
+
+    def _parse_pre_answered(self, args, value, args_file):
+        args = urllib.parse.parse_qs(args or "", keep_blank_values=True)
+        self.args = {key: ",".join(value_) for key, value_ in args.items()}
+
+        if args_file:
+            # Import YAML / JSON file but keep --args values
+            self.args = {**read_yaml(args_file), **self.args}
+
+        if value is not None:
+            self.args = {self.filter_key.split(".")[-1]: value}
 
     def _apply(self):
         logger.info("Saving the new configuration...")
@@ -567,7 +620,7 @@ class ConfigPanel:
         if not os.path.exists(dir_path):
             mkdir(dir_path, mode=0o700)
 
-        values_to_save = {**self.values, **self.new_values}
+        values_to_save = self.future_values
         if self.save_mode == "diff":
             defaults = self._get_default_values()
             values_to_save = {
@@ -590,8 +643,8 @@ class ConfigPanel:
         if services_to_reload:
             logger.info("Reloading services...")
         for service in services_to_reload:
-            if hasattr(self, "app"):
-                service = service.replace("__APP__", self.app)
+            if hasattr(self, "entity"):
+                service = service.replace("__APP__", self.entity)
             service_reload_or_restart(service)
 
     def _iterate(self, trigger=["option"]):
@@ -610,13 +663,16 @@ class Question(object):
     hide_user_input_in_prompt = False
     pattern: Optional[Dict] = None
 
-    def __init__(self, question: Dict[str, Any], context: Mapping[str, Any] = {}):
+    def __init__(self, question: Dict[str, Any],
+                 context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
         self.name = question["name"]
+        self.context = context
+        self.hooks = hooks
         self.type = question.get("type", "string")
         self.default = question.get("default", None)
         self.optional = question.get("optional", False)
         self.visible = question.get("visible", None)
-        self.context = context
         self.choices = question.get("choices", [])
         self.pattern = question.get("pattern", self.pattern)
         self.ask = question.get("ask", {"en": self.name})
@@ -626,6 +682,8 @@ class Question(object):
         self.current_value = question.get("current_value")
         # .value is the "proposed" value which we got from the user
         self.value = question.get("value")
+        # Use to return several values in case answer is in mutipart
+        self.values = {}
 
         # Empty value is parsed as empty string
         if self.default == "":
@@ -666,8 +724,8 @@ class Question(object):
             # - we doesn't want to give a specific value
             # - we want to keep the previous value
             # - we want the default value
-            self.value = None
-            return { self.name: self.value }
+            self.value = self.values[self.name] = None
+            return self.values
 
         for i in range(5):
             # Display question if no value filled or if it's a readonly message
@@ -705,14 +763,14 @@ class Question(object):
 
             break
 
-        self.value = self._post_parse_value()
+        self.value = self.values[self.name] = self._post_parse_value()
 
         # Search for post actions in hooks
-        post_hook = f"post_parse__{self.name}"
+        post_hook = f"post_ask__{self.name}"
         if post_hook in self.hooks:
-            self.hooks[post_hook](self)
+            self.values.update(self.hooks[post_hook](self))
 
-        return { self.name: self.value }
+        return self.values
 
     def _prevalidate(self):
         if self.value in [None, ""] and not self.optional:
@@ -876,8 +934,10 @@ class PasswordQuestion(Question):
     default_value = ""
     forbidden_chars = "{}"
 
-    def __init__(self, question, context: Mapping[str, Any] = {}):
-        super().__init__(question, context)
+    def __init__(self, question,
+                 context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
+        super().__init__(question, context, hooks)
         self.redact = True
         if self.default is not None:
             raise YunohostValidationError(
@@ -995,8 +1055,9 @@ class BooleanQuestion(Question):
             choices="yes/no",
         )
 
-    def __init__(self, question, context: Mapping[str, Any] = {}):
-        super().__init__(question, context)
+    def __init__(self, question, context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
+        super().__init__(question, context, hooks)
         self.yes = question.get("yes", 1)
         self.no = question.get("no", 0)
         if self.default is None:
@@ -1016,10 +1077,11 @@ class BooleanQuestion(Question):
 class DomainQuestion(Question):
     argument_type = "domain"
 
-    def __init__(self, question, context: Mapping[str, Any] = {}):
+    def __init__(self, question, context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
         from yunohost.domain import domain_list, _get_maindomain
 
-        super().__init__(question, context)
+        super().__init__(question, context, hooks)
 
         if self.default is None:
             self.default = _get_maindomain()
@@ -1042,11 +1104,12 @@ class DomainQuestion(Question):
 class UserQuestion(Question):
     argument_type = "user"
 
-    def __init__(self, question, context: Mapping[str, Any] = {}):
+    def __init__(self, question, context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
         from yunohost.user import user_list, user_info
         from yunohost.domain import _get_maindomain
 
-        super().__init__(question, context)
+        super().__init__(question, context, hooks)
         self.choices = list(user_list()["users"].keys())
 
         if not self.choices:
@@ -1068,8 +1131,9 @@ class NumberQuestion(Question):
     argument_type = "number"
     default_value = None
 
-    def __init__(self, question, context: Mapping[str, Any] = {}):
-        super().__init__(question, context)
+    def __init__(self, question, context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
+        super().__init__(question, context, hooks)
         self.min = question.get("min", None)
         self.max = question.get("max", None)
         self.step = question.get("step", None)
@@ -1120,8 +1184,9 @@ class DisplayTextQuestion(Question):
     argument_type = "display_text"
     readonly = True
 
-    def __init__(self, question, context: Mapping[str, Any] = {}):
-        super().__init__(question, context)
+    def __init__(self, question, context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
+        super().__init__(question, context, hooks)
 
         self.optional = True
         self.style = question.get(
@@ -1155,8 +1220,9 @@ class FileQuestion(Question):
             if os.path.exists(upload_dir):
                 shutil.rmtree(upload_dir)
 
-    def __init__(self, question, context: Mapping[str, Any] = {}):
-        super().__init__(question, context)
+    def __init__(self, question, context: Mapping[str, Any] = {},
+                 hooks: Dict[str, Callable] = {}):
+        super().__init__(question, context, hooks)
         self.accept = question.get("accept", "")
 
     def _prevalidate(self):
@@ -1226,7 +1292,9 @@ ARGUMENTS_TYPE_PARSERS = {
 
 
 def ask_questions_and_parse_answers(
-    raw_questions: Dict, prefilled_answers: Union[str, Mapping[str, Any]] = {},
+    raw_questions: Dict,
+    prefilled_answers: Union[str, Mapping[str, Any]] = {},
+    current_values: Union[str, Mapping[str, Any]] = {},
     hooks: Dict[str, Callable[[], None]] = {}
 ) -> List[Question]:
     """Parse arguments store in either manifest.json or actions.json or from a
@@ -1254,14 +1322,16 @@ def ask_questions_and_parse_answers(
     else:
         answers = {}
 
-
+    context = {**current_values, **answers}
     out = []
 
     for raw_question in raw_questions:
         question_class = ARGUMENTS_TYPE_PARSERS[raw_question.get("type", "string")]
         raw_question["value"] = answers.get(raw_question["name"])
-        question = question_class(raw_question, context=answers)
-        answers.update(question.ask_if_needed())
+        question = question_class(raw_question, context=context, hooks=hooks)
+        new_values = question.ask_if_needed()
+        answers.update(new_values)
+        context.update(new_values)
         out.append(question)
 
     return out

From 6f8200d9b176c211e7d2fefc232ebef898e15a23 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Sun, 24 Oct 2021 23:42:45 +0200
Subject: [PATCH 26/48] [wip] Implementation borg method + timer

---
 data/actionsmap/yunohost.yml      |  139 +++-
 data/other/config_repository.toml |   24 +-
 locales/en.json                   |    8 +-
 src/yunohost/backup.py            | 1230 +++++------------------------
 src/yunohost/repositories/borg.py |  225 ++++++
 src/yunohost/repositories/hook.py |  128 +++
 src/yunohost/repositories/tar.py  |  223 ++++++
 src/yunohost/repository.py        |  776 +++++++++++++-----
 src/yunohost/utils/network.py     |   47 ++
 9 files changed, 1525 insertions(+), 1275 deletions(-)
 create mode 100644 src/yunohost/repositories/borg.py
 create mode 100644 src/yunohost/repositories/hook.py
 create mode 100644 src/yunohost/repositories/tar.py

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 99ce3158f..d3633bfac 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -1046,11 +1046,9 @@ backup:
                 -d:
                     full: --description
                     help: Short description of the backup
-                -o:
-                    full: --output-directory
-                    help: Output directory for the backup
-                --methods:
-                    help: List of backup methods to apply (copy or tar by default)
+                -r:
+                    full: --repositories
+                    help: List of repositories where send backup files (local borg repo use by default)
                     nargs: "*"
                 --system:
                     help: List of system parts to backup (or all if none given).
@@ -1085,7 +1083,7 @@ backup:
             api: GET /backups
             arguments:
                 -r:
-                    full: --repos
+                    full: --repositories
                     help: List archives in these repositories
                     nargs: "*"
                 -i:
@@ -1120,7 +1118,7 @@ backup:
             arguments:
                 name:
                     help: Name of the local backup archive
-
+        
         ### backup_delete()
         delete:
             action_help: Delete a backup archive
@@ -1139,7 +1137,7 @@ backup:
                 ### backup_repository_list()
                 list:
                     action_help: List available repositories where put archives
-                    api: GET /backup/repositories
+                    api: GET /backups/repositories
                     arguments:
                         --full:
                             help: Show more details
@@ -1148,8 +1146,14 @@ backup:
                 ### backup_repository_info()
                 info:
                     action_help: Show info about a repository
-                    api: GET /backup/repository
+                    api: GET /backups/repository/<shortname>
                     arguments:
+                        shortname:
+                            help: ID of the repository
+                            extra:
+                                pattern: &pattern_backup_repository_shortname
+                                    - !!str ^[a-zA-Z0-9-_\.]+$
+                                    - "pattern_backup_repository_shortname"
                         -H:
                             full: --human-readable
                             help: Print sizes in human readable format
@@ -1161,14 +1165,12 @@ backup:
                 ### backup_repository_add()
                 add:
                     action_help: Add a backup repository
-                    api: POST /backup/repository/<shortname>
+                    api: POST /backups/repository/<shortname>
                     arguments:
                         shortname:
                             help: ID of the repository
                             extra:
-                                pattern: &pattern_backup_repository_shortname
-                                    - !!str ^[a-zA-Z0-9-_]+$
-                                    - "pattern_backup_repository_shortname"
+                                pattern: *pattern_backup_repository_shortname
                         -n:
                             full: --name
                             help: Short description of the repository
@@ -1182,7 +1184,6 @@ backup:
                         -m:
                             full: --method
                             help: By default 'borg' method is used, could be 'tar' or a custom method
-                            default: borg
                         -q:
                             full: --quota
                             help: Quota to configure with this repository
@@ -1197,14 +1198,13 @@ backup:
                         -d:
                             full: --alert-delay
                             help: Inactivity delay in days after which we sent alerts mails
-                            default: 7
 
                 ### backup_repository_update()
                 update:
                     action_help: Update a backup repository
-                    api: PUT /backup/repository/<name>
+                    api: PUT /backups/repository/<shortname>
                     arguments:
-                        name:
+                        shortname:
                             help: Name of the backup repository to update
                             extra:
                                 pattern: *pattern_backup_repository_shortname
@@ -1224,15 +1224,114 @@ backup:
                 ### backup_repository_remove()
                 remove:
                     action_help: Remove a backup repository
-                    api: DELETE /backup/repository/<name>
+                    api: DELETE /backups/repository/<shortname>
                     arguments:
-                        name:
+                        shortname:
                             help: Name of the backup repository to remove
                             extra:
                                 pattern: *pattern_backup_repository_shortname
                         --purge:
                             help: Remove all archives and data inside repository
-                            action: store_false
+                            action: store_true
+        timer:
+            subcategory_help: Manage backup timer
+            actions:
+                
+                ### backup_timer_list()
+                list:
+                    action_help: List backup timer
+                    api: GET /backup/timer
+                    arguments:
+                        -r:
+                            full: --repositories
+                            help: List archives in these repositories
+                            nargs: "*"
+
+                ### backup_timer_add()
+                add:
+                    action_help: Add a backup timer
+                    api: POST /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
+                            extra:
+                                pattern: &pattern_backup_archive_name
+                                    - !!str ^[\w\-\._]{1,50}(?<!\.)$
+                                    - "pattern_backup_archive_name"
+                        -d:
+                            full: --description
+                            help: Short description of the backup
+                        -r:
+                            full: --repositories
+                            help: List of repositories where send backup files (local borg repo use by default)
+                            nargs: "*"
+                        --system:
+                            help: List of system parts to backup (or all if none given).
+                            nargs: "*"
+                        --apps:
+                            help: List of application names to backup (or all if none given)
+                            nargs: "*"
+                        --schedule:
+                            help: Regular backup frequency (see systemd OnCalendar format)
+                        --alert:
+                            help: Email to alert
+                        --keep-hourly:
+                            default: 2
+                        --keep-daily:
+                            default: 7
+                        --keep-weekly:
+                            default: 8
+                        --keep-monthly:
+                            default: 12
+
+                ### backup_timer_update()
+                update:
+                    action_help: Update a backup timer
+                    api: PUT /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
+                        -d:
+                            full: --description
+                            help: Short description of the backup
+                        -r:
+                            full: --repositories
+                            help: List of repositories where send backup files (local borg repo use by default)
+                            nargs: "*"
+                        --system:
+                            help: List of system parts to backup (or all if none given).
+                            nargs: "*"
+                        --apps:
+                            help: List of application names to backup (or all if none given)
+                            nargs: "*"
+                        --schedule:
+                            help: Regular backup frequency (see systemd OnCalendar format)
+                        --alert:
+                            help: Email to alert
+                        --keep-hourly:
+                            default: 2
+                        --keep-daily:
+                            default: 7
+                        --keep-weekly:
+                            default: 8
+                        --keep-monthly:
+                            default: 12
+
+                ### backup_timer_remove()
+                remove:
+                    action_help: Remove a backup timer
+                    api: DELETE /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
+
+                ### backup_timer_info()
+                info:
+                    action_help: Get info about a backup timer
+                    api: GET /backup/timer/<name>
+                    arguments:
+                        name:
+                            help: Short prefix of the backup archives
 
 #############################
 #         Settings          #
diff --git a/data/other/config_repository.toml b/data/other/config_repository.toml
index 76c7fa987..cc0c5290f 100644
--- a/data/other/config_repository.toml
+++ b/data/other/config_repository.toml
@@ -18,60 +18,60 @@ name.en = ""
         visible = "creation"
         default = "no"
 
-        [main.main.location]
+        [main.main.domain]
         type = "string"
         visible = "creation && is_remote"
         pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        pattern.error = 'location_error' # TODO "Please provide a valid domain"
+        pattern.error = 'domain_error' # TODO "Please provide a valid domain"
         default = ""
         # FIXME: can't be a domain of this instances ?
 
-        [main.main.is_f2f]
+        [main.main.is_shf]
         help = ""
         type = "boolean"
         yes = true
         no = false
         visible = "creation && is_remote"
-        default = "no"
+        default = false
 
         [main.main.public_key]
         type = "alert"
         style = "info"
-        visible = "creation && is_remote && ! is_f2f"
+        visible = "creation && is_remote && ! is_shf"
 
         [main.main.alert]
         help = ''
         type = "tags"
-        visible = "is_remote && is_f2f"
+        visible = "is_remote && is_shf"
         pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        pattern.error = "It seems it's not a valid email"
+        pattern.error = "alert_error"
         default = []
         # "value": alert,
 
         [main.main.alert_delay]
         help = ''
         type = "number"
-        visible = "is_remote && is_f2f"
+        visible = "is_remote && is_shf"
         min = 1
         default = 7
 
         [main.main.quota]
         type = "string"
-        visible = "is_remote && is_f2f"
+        visible = "is_remote && is_shf"
         pattern.regexp = '^\d+[MGT]$'
         pattern.error = '' # TODO ""
         default = ""
 
         [main.main.port]
         type = "number"
-        visible = "is_remote && !is_f2f"
+        visible = "is_remote && !is_shf"
         min = 1
         max = 65535
         default = 22
 
         [main.main.user]
         type = "string"
-        visible = "is_remote && !is_f2f"
+        visible = "is_remote && !is_shf"
         default = ""
 
         [main.main.method]
@@ -84,6 +84,6 @@ name.en = ""
         
         [main.main.path]
         type = "path"
-        visible = "!is_remote or (is_remote and !is_f2f)"
+        visible = "!is_remote or (is_remote and !is_shf)"
         default = "/home/yunohost.backup/archives"
 
diff --git a/locales/en.json b/locales/en.json
index 68c8ec274..7f3bd6efc 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -438,6 +438,7 @@
     "log_app_upgrade": "Upgrade the '{}' app",
     "log_available_on_yunopaste": "This log is now available via {url}",
     "log_backup_create": "Create a backup archive",
+    "log_backup_repository_add": "Add a backup repository",
     "log_backup_restore_app": "Restore '{}' from a backup archive",
     "log_backup_restore_system": "Restore system from a backup archive",
     "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'",
@@ -595,9 +596,9 @@
     "regex_with_only_domain": "You can't use a regex for domain, only for path",
     "repository_config_description": "Long name",
     "repository_config_is_remote": "Remote repository",
-    "repository_config_is_f2f": "It's a YunoHost",
-    "repository_config_is_f2f_help": "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
-    "repository_config_location": "Remote server domain",
+    "repository_config_is_shf": "It's a YunoHost",
+    "repository_config_is_shf_help": "Answer yes if the remote server is a YunoHost instance or an other F2F compatible provider",
+    "repository_config_domain": "Remote server domain",
     "repository_config_public_key": "Public key to give to your BorgBackup  provider : {public_key}",
     "repository_config_alert": "Alert emails",
     "repository_config_alert_help": "Declare emails to which sent inactivity alerts",
@@ -608,6 +609,7 @@
     "repository_config_user": "User",
     "repository_config_method": "Method",
     "repository_config_path": "Archive path",
+    "repository_removed": "Repository '{repository}' removed",
     "restore_already_installed_app": "An app with the ID '{app}' is already installed",
     "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}",
     "restore_backup_too_old": "This backup archive can not be restored because it comes from a too-old YunoHost version.",
diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py
index 13b63eadb..c4dbb8cee 100644
--- a/src/yunohost/backup.py
+++ b/src/yunohost/backup.py
@@ -260,7 +260,7 @@ class BackupManager:
         backup_manager.backup()
     """
 
-    def __init__(self, name=None, description="", methods=[], work_dir=None):
+    def __init__(self, name=None, description="", repositories=[], work_dir=None):
         """
         BackupManager constructor
 
@@ -275,6 +275,7 @@ class BackupManager:
                         temporary work_dir will be created (default: None)
         """
         self.description = description or ""
+        self.repositories = repositories
         self.created_at = int(time.time())
         self.apps_return = {}
         self.system_return = {}
@@ -293,13 +294,9 @@ class BackupManager:
             self.work_dir = os.path.join(BACKUP_PATH, "tmp", name)
         self._init_work_dir()
 
-        # Initialize backup methods
-        self.methods = [
-            BackupMethod.create(method, self, repo=work_dir) for method in methods
-        ]
 
     #
-    # Misc helpers                                                          #
+    # Misc helpers
     #
 
     @property
@@ -761,10 +758,11 @@ class BackupManager:
     def backup(self):
         """Apply backup methods"""
 
-        for method in self.methods:
-            logger.debug(m18n.n("backup_applying_method_" + method.method_name))
-            method.mount_and_backup()
-            logger.debug(m18n.n("backup_method_" + method.method_name + "_finished"))
+        for repo in self.repositories:
+            logger.debug(m18n.n("backup_applying_method_" + repo.method_name))
+            archive = BackupArchive(repo, name=self.name, manager=self)
+            archive.organize_and_backup()
+            logger.debug(m18n.n("backup_method_" + repo.method_name + "_finished"))
 
     def _compute_backup_size(self):
         """
@@ -1050,7 +1048,7 @@ class RestoreManager:
     # Archive mounting                                                      #
     #
 
-    def mount(self):
+    def extract(self):
         """
         Mount the archive. We avoid copy to be able to restore on system without
         too many space.
@@ -1081,7 +1079,7 @@ class RestoreManager:
 
         filesystem.mkdir(self.work_dir, parents=True)
 
-        self.method.mount()
+        self.method.extract()
 
         self._read_info_files()
 
@@ -1566,881 +1564,6 @@ class RestoreManager:
                 logger.error(failure_message_with_debug_instructions)
 
 
-#
-# Backup methods                                                            #
-#
-class BackupMethod(object):
-
-    """
-    BackupMethod is an abstract class that represents a way to backup and
-    restore a list of files.
-
-    Daughters of this class can be used by a BackupManager or RestoreManager
-    instance. Some methods are meant to be used by BackupManager and others
-    by RestoreManager.
-
-    BackupMethod has a factory method "create" to initialize instances.
-
-    Currently, there are 3 BackupMethods implemented:
-
-    CopyBackupMethod
-    ----------------
-    This method corresponds to a raw (uncompressed) copy of files to a location,
-    and (could?) reverse the copy when restoring.
-
-    TarBackupMethod
-    ---------------
-    This method compresses all files to backup in a .tar archive. When
-    restoring, it untars the required parts.
-
-    CustomBackupMethod
-    ------------------
-    This one use a custom bash scrip/hook "backup_method" to do the
-    backup/restore operations. A user can add his own hook inside
-    /etc/yunohost/hooks.d/backup_method/
-
-    Public properties:
-        method_name
-
-    Public methods:
-        mount_and_backup(self)
-        mount(self)
-        create(cls, method, **kwargs)
-        info(archive_name)
-
-    Usage:
-        method = BackupMethod.create("tar", backup_manager)
-        method.mount_and_backup()
-        #or
-        method = BackupMethod.create("copy", restore_manager)
-        method.mount()
-    """
-
-    @classmethod
-    def create(cls, method, manager, **kwargs):
-        """
-        Factory method to create instance of BackupMethod
-
-        Args:
-        method -- (string) The method name of an existing BackupMethod. If the
-        name is unknown the CustomBackupMethod will be tried
-        *args    -- Specific args for the method, could be the repo target by the
-        method
-
-        Return a BackupMethod instance
-        """
-        known_methods = {c.method_name: c for c in BackupMethod.__subclasses__()}
-        backup_method = known_methods.get(method, CustomBackupMethod)
-        return backup_method(manager, method=method, **kwargs)
-
-    def __init__(self, manager, repo=None, **kwargs):
-        """
-        BackupMethod constructors
-
-        Note it is an abstract class. You should use the "create" class method
-        to create instance.
-
-        Args:
-           repo -- (string|None) A string that represent the repo where put or
-                   get the backup. It could be a path, and in future a
-                   BackupRepository object. If None, the default repo is used :
-                   /home/yunohost.backup/archives/
-        """
-        self.manager = manager
-        if not repo or isinstance(repo, basestring):
-            repo = BackupRepository.get(ARCHIVES_PATH)
-        self.repo = repo
-
-    @property
-    def method_name(self):
-        """Return the string name of a BackupMethod (eg "tar" or "copy")"""
-        raise YunohostError("backup_abstract_method")
-
-    @property
-    def archive_path(self):
-        """Return the archive path"""
-        return self.repo.location + '::' + self.name
-
-    @property
-    def name(self):
-        """Return the backup name"""
-        return self.manager.name
-
-    @property
-    def work_dir(self):
-        """
-        Return the working directory
-
-        For a BackupManager, it is the directory where we prepare the files to
-        backup
-
-        For a RestoreManager, it is the directory where we mount the archive
-        before restoring
-        """
-        return self.manager.work_dir
-
-    def need_mount(self):
-        """
-        Return True if this backup method need to organize path to backup by
-        binding its in the working directory before to backup its.
-
-        Indeed, some methods like tar or copy method don't need to organize
-        files before to add it inside the archive, but others like borgbackup
-        are not able to organize directly the files. In this case we have the
-        choice to organize in the working directory before to put in the archive
-        or to organize after mounting the archive before the restoring
-        operation.
-
-        The default behaviour is to return False. To change it override the
-        method.
-
-        Note it's not a property because some overrided methods could do long
-        treatment to get this info
-        """
-        return False
-
-    def mount_and_backup(self):
-        """
-        Run the backup on files listed by  the BackupManager instance
-
-        This method shouldn't be overrided, prefer overriding self.backup() and
-        self.clean()
-        """
-        if self.need_mount():
-            self._organize_files()
-
-        try:
-            self.backup()
-        finally:
-            self.clean()
-
-    def mount(self):
-        """
-        Mount the archive from RestoreManager instance in the working directory
-
-        This method should be extended.
-        """
-        pass
-
-    def info(self, name):
-        self._assert_archive_exists()
-
-        info_json = self._get_info_string()
-        if not self._info_json:
-            raise YunohostError('backup_info_json_not_implemented')
-        try:
-            info = json.load(info_json)
-        except:
-            logger.debug("unable to load info json", exc_info=1)
-            raise YunohostError('backup_invalid_archive')
-
-        # (legacy) Retrieve backup size
-        # FIXME
-        size = info.get("size", 0)
-        if not size:
-            tar = tarfile.open(
-                archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
-            )
-            size = reduce(
-                lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
-            )
-            tar.close()
-
-        return info
-
-    def clean(self):
-        """
-        Umount sub directories of working dirextories and delete it if temporary
-        """
-        if self.need_mount():
-            if not _recursive_umount(self.work_dir):
-                raise YunohostError("backup_cleaning_failed")
-
-        if self.manager.is_tmp_work_dir:
-            filesystem.rm(self.work_dir, True, True)
-
-    def _check_is_enough_free_space(self):
-        """
-        Check free space in repository or output directory before to backup
-        """
-        # TODO How to do with distant repo or with deduplicated backup ?
-        backup_size = self.manager.size
-
-        free_space = free_space_in_directory(self.repo)
-
-        if free_space < backup_size:
-            logger.debug(
-                "Not enough space at %s (free: %s / needed: %d)",
-                self.repo,
-                free_space,
-                backup_size,
-            )
-            raise YunohostValidationError("not_enough_disk_space", path=self.repo)
-
-    def _organize_files(self):
-        """
-        Mount all csv src in their related path
-
-        The goal is to organize the files app by app and hook by hook, before
-        custom backup method or before the restore operation (in the case of an
-        unorganize archive).
-
-        The usage of binding could be strange for a user because the du -sb
-        command will return that the working directory is big.
-        """
-        paths_needed_to_be_copied = []
-        for path in self.manager.paths_to_backup:
-            src = path["source"]
-
-            if self.manager is RestoreManager:
-                # TODO Support to run this before a restore (and not only before
-                # backup). To do that RestoreManager.unorganized_work_dir should
-                # be implemented
-                src = os.path.join(self.unorganized_work_dir, src)
-
-            dest = os.path.join(self.work_dir, path["dest"])
-            if dest == src:
-                continue
-            dest_dir = os.path.dirname(dest)
-
-            # Be sure the parent dir of destination exists
-            if not os.path.isdir(dest_dir):
-                filesystem.mkdir(dest_dir, parents=True)
-
-            # For directory, attempt to mount bind
-            if os.path.isdir(src):
-                filesystem.mkdir(dest, parents=True, force=True)
-
-                try:
-                    subprocess.check_call(["mount", "--rbind", src, dest])
-                    subprocess.check_call(["mount", "-o", "remount,ro,bind", dest])
-                except Exception:
-                    logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest))
-                    # To check if dest is mounted, use /proc/mounts that
-                    # escape spaces as \040
-                    raw_mounts = read_file("/proc/mounts").strip().split("\n")
-                    mounts = [m.split()[1] for m in raw_mounts]
-                    mounts = [m.replace("\\040", " ") for m in mounts]
-                    if dest in mounts:
-                        subprocess.check_call(["umount", "-R", dest])
-                else:
-                    # Success, go to next file to organize
-                    continue
-
-            # For files, create a hardlink
-            elif os.path.isfile(src) or os.path.islink(src):
-                # Can create a hard link only if files are on the same fs
-                # (i.e. we can't if it's on a different fs)
-                if os.stat(src).st_dev == os.stat(dest_dir).st_dev:
-                    # Don't hardlink /etc/cron.d files to avoid cron bug
-                    # 'NUMBER OF HARD LINKS > 1' see #1043
-                    cron_path = os.path.abspath("/etc/cron") + "."
-                    if not os.path.abspath(src).startswith(cron_path):
-                        try:
-                            os.link(src, dest)
-                        except Exception as e:
-                            # This kind of situation may happen when src and dest are on different
-                            # logical volume ... even though the st_dev check previously match...
-                            # E.g. this happens when running an encrypted hard drive
-                            # where everything is mapped to /dev/mapper/some-stuff
-                            # yet there are different devices behind it or idk ...
-                            logger.warning(
-                                "Could not link %s to %s (%s) ... falling back to regular copy."
-                                % (src, dest, str(e))
-                            )
-                        else:
-                            # Success, go to next file to organize
-                            continue
-
-            # If mountbind or hardlink couldnt be created,
-            # prepare a list of files that need to be copied
-            paths_needed_to_be_copied.append(path)
-
-        if len(paths_needed_to_be_copied) == 0:
-            return
-        # Manage the case where we are not able to use mount bind abilities
-        # It could be just for some small files on different filesystems or due
-        # to mounting error
-
-        # Compute size to copy
-        size = sum(disk_usage(path["source"]) for path in paths_needed_to_be_copied)
-        size /= 1024 * 1024  # Convert bytes to megabytes
-
-        # Ask confirmation for copying
-        if size > MB_ALLOWED_TO_ORGANIZE:
-            try:
-                i = Moulinette.prompt(
-                    m18n.n(
-                        "backup_ask_for_copying_if_needed",
-                        answers="y/N",
-                        size=str(size),
-                    )
-                )
-            except NotImplemented:
-                raise YunohostError("backup_unable_to_organize_files")
-            else:
-                if i != "y" and i != "Y":
-                    raise YunohostError("backup_unable_to_organize_files")
-
-        # Copy unbinded path
-        logger.debug(m18n.n("backup_copying_to_organize_the_archive", size=str(size)))
-        for path in paths_needed_to_be_copied:
-            dest = os.path.join(self.work_dir, path["dest"])
-            if os.path.isdir(path["source"]):
-                shutil.copytree(path["source"], dest, symlinks=True)
-            else:
-                shutil.copy(path["source"], dest)
-
-
-class CopyBackupMethod(BackupMethod):
-
-    """
-    This class just do an uncompress copy of each file in a location, and
-    could be the inverse for restoring
-    """
-
-    # FIXME: filesystem.mkdir(self.repo.path, parent=True)
-
-    method_name = "copy"
-
-    def backup(self):
-        """Copy prepared files into a the repo"""
-        # Check free space in output
-        self._check_is_enough_free_space()
-
-        for path in self.manager.paths_to_backup:
-            source = path["source"]
-            dest = os.path.join(self.repo.path, path["dest"])
-            if source == dest:
-                logger.debug("Files already copyed")
-                return
-
-            dest_parent = os.path.dirname(dest)
-            if not os.path.exists(dest_parent):
-                filesystem.mkdir(dest_parent, 0o700, True, uid="admin")
-
-            if os.path.isdir(source):
-                shutil.copytree(source, dest)
-            else:
-                shutil.copy(source, dest)
-
-    def mount(self):
-        """
-        Mount the uncompress backup in readonly mode to the working directory
-        """
-        # FIXME: This code is untested because there is no way to run it from
-        # the ynh cli
-        super(CopyBackupMethod, self).mount()
-
-        if not os.path.isdir(self.repo.path):
-            raise YunohostError("backup_no_uncompress_archive_dir")
-
-        filesystem.mkdir(self.work_dir, parent=True)
-        ret = subprocess.call(["mount", "-r", "--rbind", self.repo.path,
-                              self.work_dir])
-        if ret == 0:
-            return
-        else:
-            logger.warning(m18n.n("bind_mouting_disable"))
-            subprocess.call(["mountpoint", "-q", self.repo.path,
-                            "&&", "umount", "-R", self.repo.path])
-            raise YunohostError("backup_cant_mount_uncompress_archive")
-
-        logger.warning(
-            "Could not mount the backup in readonly mode with --rbind ... Unmounting"
-        )
-        # FIXME : Does this stuff really works ? '&&' is going to be interpreted as an argument for mounpoint here ... Not as a classical '&&' ...
-        subprocess.call(
-            ["mountpoint", "-q", self.work_dir, "&&", "umount", "-R", self.work_dir]
-        )
-        raise YunohostError("backup_cant_mount_uncompress_archive")
-
-    def copy(self, file, target):
-        shutil.copy(file, target)
-
-
-class TarBackupMethod(BackupMethod):
-
-    # FIXME: filesystem.mkdir(self.repo.path, parent=True)
-    method_name = "tar"
-
-    @property
-    def archive_path(self):
-
-        if isinstance(self.manager, BackupManager) and settings_get(
-            "backup.compress_tar_archives"
-        ):
-            return os.path.join(self.repo.path, self.name + ".tar.gz")
-
-        f = os.path.join(self.repo.path, self.name + ".tar")
-        if os.path.exists(f + ".gz"):
-            f += ".gz"
-        return f
-
-    def backup(self):
-        """
-        Compress prepared files
-
-        It adds the info.json in /home/yunohost.backup/archives and if the
-        compress archive isn't located here, add a symlink to the archive to.
-        """
-
-        if not os.path.exists(self.repo.path):
-            filesystem.mkdir(self.repo.path, 0o750, parents=True, uid="admin")
-
-        # Check free space in output
-        self._check_is_enough_free_space()
-
-        # Open archive file for writing
-        try:
-            tar = tarfile.open(
-                self.archive_path,
-                "w:gz" if self.archive_path.endswith(".gz") else "w",
-            )
-        except Exception:
-            logger.debug(
-                "unable to open '%s' for writing", self.archive_path, exc_info=1
-            )
-            raise YunohostError("backup_archive_open_failed")
-
-        # Add files to the archive
-        try:
-            for path in self.manager.paths_to_backup:
-                # Add the "source" into the archive and transform the path into
-                # "dest"
-                tar.add(path["source"], arcname=path["dest"])
-        except IOError:
-            logger.error(
-                m18n.n(
-                    "backup_archive_writing_error",
-                    source=path["source"],
-                    archive=self._archive_file,
-                    dest=path["dest"],
-                ),
-                exc_info=1,
-            )
-            raise YunohostError("backup_creation_failed")
-        finally:
-            tar.close()
-
-        # Move info file
-        shutil.copy(
-            os.path.join(self.work_dir, "info.json"),
-            os.path.join(ARCHIVES_PATH, self.name + ".info.json"),
-        )
-
-        # If backuped to a non-default location, keep a symlink of the archive
-        # to that location
-        link = os.path.join(self.repo.path, self.name + ".tar")
-        if not os.path.isfile(link):
-            os.symlink(self.archive_path, link)
-
-    def mount(self):
-        """
-        Mount the archive. We avoid intermediate copies to be able to restore on system with low free space.
-        """
-        super(TarBackupMethod, self).mount()
-
-        # Mount the tarball
-        logger.debug(m18n.n("restore_extracting"))
-        try:
-            tar = tarfile.open(
-                self.archive_path,
-                "r:gz" if self.archive_path.endswith(".gz") else "r",
-            )
-        except Exception:
-            logger.debug(
-                "cannot open backup archive '%s'", self.archive_path, exc_info=1
-            )
-            raise YunohostError("backup_archive_open_failed")
-
-        try:
-            files_in_archive = tar.getnames()
-        except (IOError, EOFError, tarfile.ReadError) as e:
-            raise YunohostError(
-                "backup_archive_corrupted", archive=self.archive_path, error=str(e)
-            )
-
-        if "info.json" in tar.getnames():
-            leading_dot = ""
-            tar.extract("info.json", path=self.work_dir)
-        elif "./info.json" in files_in_archive:
-            leading_dot = "./"
-            tar.extract("./info.json", path=self.work_dir)
-        else:
-            logger.debug(
-                "unable to retrieve 'info.json' inside the archive", exc_info=1
-            )
-            tar.close()
-            raise YunohostError(
-                "backup_archive_cant_retrieve_info_json", archive=self.archive_path
-            )
-
-        if "backup.csv" in files_in_archive:
-            tar.extract("backup.csv", path=self.work_dir)
-        elif "./backup.csv" in files_in_archive:
-            tar.extract("./backup.csv", path=self.work_dir)
-        else:
-            # Old backup archive have no backup.csv file
-            pass
-
-        # Extract system parts backup
-        conf_extracted = False
-
-        system_targets = self.manager.targets.list("system", exclude=["Skipped"])
-        apps_targets = self.manager.targets.list("apps", exclude=["Skipped"])
-
-        for system_part in system_targets:
-            # Caution: conf_ynh_currenthost helpers put its files in
-            # conf/ynh
-            if system_part.startswith("conf_"):
-                if conf_extracted:
-                    continue
-                system_part = "conf/"
-                conf_extracted = True
-            else:
-                system_part = system_part.replace("_", "/") + "/"
-            subdir_and_files = [
-                tarinfo
-                for tarinfo in tar.getmembers()
-                if tarinfo.name.startswith(leading_dot + system_part)
-            ]
-            tar.extractall(members=subdir_and_files, path=self.work_dir)
-        subdir_and_files = [
-            tarinfo
-            for tarinfo in tar.getmembers()
-            if tarinfo.name.startswith(leading_dot + "hooks/restore/")
-        ]
-        tar.extractall(members=subdir_and_files, path=self.work_dir)
-
-        # Extract apps backup
-        for app in apps_targets:
-            subdir_and_files = [
-                tarinfo
-                for tarinfo in tar.getmembers()
-                if tarinfo.name.startswith(leading_dot + "apps/" + app)
-            ]
-            tar.extractall(members=subdir_and_files, path=self.work_dir)
-
-        tar.close()
-
-    def copy(self, file, target):
-        tar = tarfile.open(
-            self._archive_file, "r:gz" if self._archive_file.endswith(".gz") else "r"
-        )
-        file_to_extract = tar.getmember(file)
-        # Remove the path
-        file_to_extract.name = os.path.basename(file_to_extract.name)
-        tar.extract(file_to_extract, path=target)
-        tar.close()
-
-    def list(self):
-        # Get local archives sorted according to last modification time
-        # (we do a realpath() to resolve symlinks)
-        archives = glob("%s/*.tar.gz" % self.repo.path) + glob("%s/*.tar" % self.repo.path)
-        archives = set([os.path.realpath(archive) for archive in archives])
-        archives = sorted(archives, key=lambda x: os.path.getctime(x))
-        # Extract only filename without the extension
-
-        def remove_extension(f):
-            if f.endswith(".tar.gz"):
-                return os.path.basename(f)[: -len(".tar.gz")]
-            else:
-                return os.path.basename(f)[: -len(".tar")]
-
-        return [remove_extension(f) for f in archives]
-
-
-    def _archive_exists(self):
-        return os.path.lexists(self.archive_path)
-
-    def _assert_archive_exists(self):
-        if not self._archive_exists():
-            raise YunohostError('backup_archive_name_unknown', name=self.name)
-
-        # If symlink, retrieve the real path
-        if os.path.islink(self.archive_path):
-            archive_file = os.path.realpath(self.archive_path)
-
-            # Raise exception if link is broken (e.g. on unmounted external storage)
-            if not os.path.exists(archive_file):
-                raise YunohostError('backup_archive_broken_link',
-                                    path=archive_file)
-
-    def _get_info_string(self):
-        name = self.name
-        if name.endswith(".tar.gz"):
-            name = name[: -len(".tar.gz")]
-        elif name.endswith(".tar"):
-            name = name[: -len(".tar")]
-
-        archive_file = "%s/%s.tar" % (self.repo.path, name)
-
-        # Check file exist (even if it's a broken symlink)
-        if not os.path.lexists(archive_file):
-            archive_file += ".gz"
-            if not os.path.lexists(archive_file):
-                raise YunohostValidationError("backup_archive_name_unknown", name=name)
-
-        # If symlink, retrieve the real path
-        if os.path.islink(archive_file):
-            archive_file = os.path.realpath(archive_file)
-
-            # Raise exception if link is broken (e.g. on unmounted external storage)
-            if not os.path.exists(archive_file):
-                raise YunohostValidationError(
-                    "backup_archive_broken_link", path=archive_file
-                )
-
-        info_file = "%s/%s.info.json" % (self.repo.path, name)
-
-        if not os.path.exists(info_file):
-            tar = tarfile.open(
-                archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
-            )
-            info_dir = info_file + ".d"
-
-            try:
-                files_in_archive = tar.getnames()
-            except (IOError, EOFError, tarfile.ReadError) as e:
-                raise YunohostError(
-                    "backup_archive_corrupted", archive=archive_file, error=str(e)
-                )
-
-            try:
-                if "info.json" in files_in_archive:
-                    tar.extract("info.json", path=info_dir)
-                elif "./info.json" in files_in_archive:
-                    tar.extract("./info.json", path=info_dir)
-                else:
-                    raise KeyError
-            except KeyError:
-                logger.debug(
-                    "unable to retrieve '%s' inside the archive", info_file, exc_info=1
-                )
-                raise YunohostError(
-                    "backup_archive_cant_retrieve_info_json", archive=archive_file
-                )
-            else:
-                shutil.move(os.path.join(info_dir, "info.json"), info_file)
-            finally:
-                tar.close()
-            os.rmdir(info_dir)
-
-        try:
-            return read_file(info_file)
-        except MoulinetteError:
-            logger.debug("unable to load '%s'", info_file, exc_info=1)
-            raise YunohostError('backup_invalid_archive')
-
-
-class BorgBackupMethod(BackupMethod):
-
-    def __init__(self, repo=None):
-        super(BorgBackupMethod, self).__init__(repo)
-
-        if not self.repo.domain:
-            filesystem.mkdir(self.repo.path, parent=True)
-
-        cmd = ['borg', 'init', self.repo.location]
-
-        if self.repo.quota:
-            cmd += ['--storage-quota', self.repo.quota]
-        self._call('init', cmd)
-
-    @property
-    def method_name(self):
-        return 'borg'
-
-
-    def need_mount(self):
-        return True
-
-    def backup(self):
-        """ Backup prepared files with borg """
-
-        cmd = ['borg', 'create', self.archive_path, './']
-        self._call('backup', cmd)
-
-    def mount(self, restore_manager):
-        """ Extract and mount needed files with borg """
-        super(BorgBackupMethod, self).mount(restore_manager)
-
-        # Export as tar needed files through a pipe
-        cmd = ['borg', 'export-tar', self.archive_path, '-']
-        borg = self._run_borg_command(cmd, stdout=subprocess.PIPE)
-
-        # And uncompress it into the working directory
-        untar = subprocess.Popen(['tar', 'x'], cwd=self.work_dir, stdin=borg.stdout)
-        borg_return_code = borg.wait()
-        untar_return_code = untar.wait()
-        if borg_return_code + untar_return_code != 0:
-            # err = untar.communicate()[1]
-            raise YunohostError('backup_borg_mount_error')
-
-    def list(self):
-        """ Return a list of archives names
-
-        Exceptions:
-        backup_borg_list_error -- Raised if the borg script failed
-        """
-        cmd = ['borg', 'list', self.repo.location, '--short']
-        out = self._call('list', cmd)
-        result = out.strip().splitlines()
-        return result
-
-    def _assert_archive_exists(self):
-        """ Trigger an error if archive is missing
-
-        Exceptions:
-        backup_borg_exist_error -- Raised if the borg script failed
-        """
-        cmd = ['borg', 'list', self.archive_path]
-        self._call('exist', cmd)
-
-    def _get_info_string(self):
-        """ Return json string of the info.json file
-
-        Exceptions:
-        backup_borg_info_error -- Raised if the custom script failed
-        """
-        cmd = ['borg', 'extract', '--stdout', self.archive_path, 'info.json']
-        return self._call('info', cmd)
-
-    def _run_borg_command(self, cmd, stdout=None):
-        """ Call a submethod of borg with the good context
-        """
-        env = dict(os.environ)
-
-        if self.repo.domain:
-            # TODO Use the best/good key
-            private_key = "/root/.ssh/ssh_host_ed25519_key"
-
-            # Don't check ssh fingerprint strictly the first time
-            # TODO improve this by publishing and checking this with DNS
-            strict = 'yes' if self.repo.domain in open('/root/.ssh/known_hosts').read() else 'no'
-            env['BORG_RSH'] = "ssh -i %s -oStrictHostKeyChecking=%s"
-            env['BORG_RSH'] = env['BORG_RSH'] % (private_key, strict)
-
-        # In case, borg need a passphrase to get access to the repo
-        if self.repo.passphrase:
-            cmd += ['-e', 'repokey']
-            env['BORG_PASSPHRASE'] = self.repo.passphrase
-
-        return subprocess.Popen(cmd, env=env, stdout=stdout)
-
-    def _call(self, action, cmd):
-        borg = self._run_borg_command(cmd)
-        return_code = borg.wait()
-        if return_code:
-            raise YunohostError('backup_borg_' + action + '_error')
-
-        out, _ = borg.communicate()
-
-        return out
-
-
-
-class CustomBackupMethod(BackupMethod):
-
-    """
-    This class use a bash script/hook "backup_method" to do the
-    backup/restore operations. A user can add his own hook inside
-    /etc/yunohost/hooks.d/backup_method/
-    """
-
-    method_name = "custom"
-
-    def __init__(self, manager, repo=None, method=None, **kwargs):
-        super(CustomBackupMethod, self).__init__(manager, repo)
-        self.args = kwargs
-        self.method = method
-        self._need_mount = None
-
-    def need_mount(self):
-        """Call the backup_method hook to know if we need to organize files"""
-        if self._need_mount is not None:
-            return self._need_mount
-
-        try:
-            self._call('nedd_mount')
-        except YunohostError:
-            return False
-        return True
-
-    def backup(self):
-        """
-        Launch a custom script to backup
-        """
-
-        self._call('backup', self.work_dir, self.name, self.repo.location, self.manager.size,
-                self.manager.description)
-
-    def mount(self):
-        """
-        Launch a custom script to mount the custom archive
-        """
-        super().mount()
-        self._call('mount', self.work_dir, self.name, self.repo.location, self.manager.size,
-                self.manager.description)
-
-    def list(self):
-        """ Return a list of archives names
-
-        Exceptions:
-        backup_custom_list_error -- Raised if the custom script failed
-        """
-        out = self._call('list', self.repo.location)
-        result = out.strip().splitlines()
-        return result
-
-    def _assert_archive_exists(self):
-        """ Trigger an error if archive is missing
-
-        Exceptions:
-        backup_custom_exist_error -- Raised if the custom script failed
-        """
-        self._call('exist', self.name, self.repo.location)
-
-    def _get_info_string(self):
-        """ Return json string of the info.json file
-
-        Exceptions:
-        backup_custom_info_error -- Raised if the custom script failed
-        """
-        return self._call('info', self.name, self.repo.location)
-
-    def _call(self, *args):
-        """ Call a submethod of backup method hook
-
-        Exceptions:
-        backup_custom_ACTION_error -- Raised if the custom script failed
-        """
-        ret = hook_callback("backup_method", [self.method],
-                            args=args)
-
-        ret_failed = [
-            hook
-            for hook, infos in ret.items()
-            if any(result["state"] == "failed" for result in infos.values())
-        ]
-        if ret_failed:
-            raise YunohostError("backup_custom_" + args[0] + "_error")
-
-        return ret["succeed"][self.method]["stdreturn"]
-
-    def _get_args(self, action):
-        """Return the arguments to give to the custom script"""
-        return [
-            action,
-            self.work_dir,
-            self.name,
-            self.repo,
-            self.manager.size,
-            self.manager.description,
-        ]
-
-
 #
 # "Front-end"                                                               #
 #
@@ -2450,7 +1573,7 @@ def backup_create(
     operation_logger,
     name=None,
     description=None,
-    repos=[],
+    reposistories=[],
     system=[],
     apps=[],
     dry_run=False,
@@ -2474,13 +1597,9 @@ def backup_create(
     #
 
     # Validate there is no archive with the same name
-    if name and name in backup_list()["archives"]:
+    if name and name in backup_list(repositories)["archives"]:
         raise YunohostValidationError("backup_archive_name_exists")
 
-    # By default we backup using the tar method
-    if not methods:
-        methods = ["tar"]
-
     # If no --system or --apps given, backup everything
     if system is None and apps is None:
         system = []
@@ -2493,18 +1612,17 @@ def backup_create(
     operation_logger.start()
 
     # Create yunohost archives directory if it does not exists
-    _create_archive_dir()
-
-    # Prepare files to backup
-    backup_manager = BackupManager(name, description)
+    _create_archive_dir() # FIXME
 
     # Add backup methods
-    if repos == []:
-        repos = ["/home/yunohost.backup/archives"]
+    if not repositories:
+        repositories = ["local-borg"]
 
-    for repo in repos:
-        repo = BackupRepository.get(repo)
-        backup_manager.add(repo.method)
+    repositories = [BackupRepository(repo) for repo in reposistories]
+
+    # Prepare files to backup
+    backup_manager = BackupManager(name, description,
+                                   repositories=repositories)
 
     # Add backup targets (system and apps)
 
@@ -2611,7 +1729,7 @@ def backup_restore(name, system=[], apps=[], force=False):
     #
 
     logger.info(m18n.n("backup_mount_archive_for_restore"))
-    restore_manager.mount()
+    restore_manager.extract()
     restore_manager.restore()
 
     # Check if something has been restored
@@ -2623,7 +1741,7 @@ def backup_restore(name, system=[], apps=[], force=False):
     return restore_manager.targets.results
 
 
-def backup_list(repos=[], with_info=False, human_readable=False):
+def backup_list(repositories=[], with_info=False, human_readable=False):
     """
     List available local backup archives
 
@@ -2633,73 +1751,25 @@ def backup_list(repos=[], with_info=False, human_readable=False):
         human_readable -- Print sizes in human readable format
 
     """
-    result = OrderedDict()
+    return {
+        name: BackupRepository(name).list(with_info)
+        for name in repositories or BackupRepository.list(full=False)
+    }
 
-    if repos == []:
-        repos = backup_repository_list(full=True)
-    else:
-        for k, repo in repos:
-            repos[k] = BackupRepository.get(repo)
+def backup_download(name, repository):
+
+    repo = BackupRepository(repo)
+    archive = BackupArchive(name, repo)
+    return archive.download()
 
 
-    for repo in repos:
-        result[repo.name] = repo.list(with_info)
+def backup_mount(name, repository, path):
 
-        # Add details
-        if result[repo.name] and with_info:
-            d = OrderedDict()
-            for a in result[repo.name]:
-                try:
-                    d[a] = backup_info(a, repo=repo.location, human_readable=human_readable)
-                except YunohostError as e:
-                    logger.warning(str(e))
-                except Exception:
-                    import traceback
+    repo = BackupRepository(repo)
+    archive = BackupArchive(name, repo)
+    return archive.mount(path)
 
-                    logger.warning(
-                        "Could not check infos for archive %s: %s"
-                        % (archive, "\n" + traceback.format_exc())
-                    )
-
-            result[repo.name] = d
-
-    return result
-
-def backup_download(name):
-    # TODO Integrate in backup methods
-    if Moulinette.interface.type != "api":
-        logger.error(
-            "This option is only meant for the API/webadmin and doesn't make sense for the command line."
-        )
-        return
-
-    archive_file = "%s/%s.tar" % (ARCHIVES_PATH, name)
-
-    # Check file exist (even if it's a broken symlink)
-    if not os.path.lexists(archive_file):
-        archive_file += ".gz"
-        if not os.path.lexists(archive_file):
-            raise YunohostValidationError("backup_archive_name_unknown", name=name)
-
-    # If symlink, retrieve the real path
-    if os.path.islink(archive_file):
-        archive_file = os.path.realpath(archive_file)
-
-        # Raise exception if link is broken (e.g. on unmounted external storage)
-        if not os.path.exists(archive_file):
-            raise YunohostValidationError(
-                "backup_archive_broken_link", path=archive_file
-            )
-
-    # We return a raw bottle HTTPresponse (instead of serializable data like
-    # list/dict, ...), which is gonna be picked and used directly by moulinette
-    from bottle import static_file
-
-    archive_folder, archive_file_name = archive_file.rsplit("/", 1)
-    return static_file(archive_file_name, archive_folder, download=archive_file_name)
-
-
-def backup_info(name, repo=None, with_details=False, human_readable=False):
+def backup_info(name, repository=None, with_details=False, human_readable=False):
     """
     Get info about a local backup archive
 
@@ -2709,57 +1779,11 @@ def backup_info(name, repo=None, with_details=False, human_readable=False):
         human_readable -- Print sizes in human readable format
 
     """
-    if not repo:
-        repo = '/home/yunohost.backup/archives/'
+    repo = BackupRepository(repo)
+    archive = BackupArchive(name, repo)
+    return archive.info()
 
-    repo = BackupRepository.get(repo)
-
-    info = repo.method.info(name)
-
-    result = {
-        "path": repo.archive_path,
-        "created_at": datetime.utcfromtimestamp(info["created_at"]),
-        "description": info["description"],
-        "size": size,
-    }
-    if human_readable:
-        result['size'] = binary_to_human(result['size']) + 'B'
-
-    if with_details:
-        system_key = "system"
-        # Historically 'system' was 'hooks'
-        if "hooks" in info.keys():
-            system_key = "hooks"
-
-        if "size_details" in info.keys():
-            for category in ["apps", "system"]:
-                for name, key_info in info[category].items():
-
-                    if category == "system":
-                        # Stupid legacy fix for weird format between 3.5 and 3.6
-                        if isinstance(key_info, dict):
-                            key_info = key_info.keys()
-                        info[category][name] = key_info = {"paths": key_info}
-                    else:
-                        info[category][name] = key_info
-
-                    if name in info["size_details"][category].keys():
-                        key_info["size"] = info["size_details"][category][name]
-                        if human_readable:
-                            key_info["size"] = binary_to_human(key_info["size"]) + "B"
-                    else:
-                        key_info["size"] = -1
-                        if human_readable:
-                            key_info["size"] = "?"
-
-        result["apps"] = info["apps"]
-        result["system"] = info[system_key]
-        result["from_yunohost_version"] = info.get("from_yunohost_version")
-    return result
-
-
-
-def backup_delete(name):
+def backup_delete(name, repository):
     """
     Delete a backup
 
@@ -2767,31 +1791,13 @@ def backup_delete(name):
         name -- Name of the local backup archive
 
     """
-    if name not in backup_list()["archives"]:
-        raise YunohostValidationError("backup_archive_name_unknown", name=name)
+    repo = BackupRepository(repo)
+    archive = BackupArchive(name, repo)
 
+    # FIXME Those are really usefull ?
     hook_callback("pre_backup_delete", args=[name])
 
-    archive_file = "%s/%s.tar" % (ARCHIVES_PATH, name)
-    if os.path.exists(archive_file + ".gz"):
-        archive_file += ".gz"
-    info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name)
-
-    files_to_delete = [archive_file, info_file]
-
-    # To handle the case where archive_file is in fact a symlink
-    if os.path.islink(archive_file):
-        actual_archive = os.path.realpath(archive_file)
-        files_to_delete.append(actual_archive)
-
-    for backup_file in files_to_delete:
-        if not os.path.exists(backup_file):
-            continue
-        try:
-            os.remove(backup_file)
-        except Exception:
-            logger.debug("unable to delete '%s'", backup_file, exc_info=1)
-            logger.warning(m18n.n("backup_delete_error", path=backup_file))
+    archive.delete()
 
     hook_callback("post_backup_delete", args=[name])
 
@@ -2802,38 +1808,156 @@ def backup_delete(name):
 #
 # Repository subcategory
 #
-import yunohost.repository
 
 
 def backup_repository_list(full=False):
-    return yunohost.repository.backup_repository_list(full)
+    """
+    List available repositories where put archives
+    """
+    return { "repositories": BackupRepository.list(full) }
 
 
-def backup_repository_info(shortname, human_readable=True, space_used=False):
-    return yunohost.repository.backup_repository_info(shortname, human_readable, space_used)
+def backup_repository_info(shortname, space_used=False):
+    return BackupRepository(shortname).info(space_used)
 
 
-def backup_repository_add(shortname, name=None, location=None,
+@is_unit_operation()
+def backup_repository_add(operation_logger, shortname, name=None, location=None,
                           method=None, quota=None, passphrase=None,
-                          alert=[], alert_delay=7):
-    return yunohost.repository.backup_repository_add(location, shortname, name, method, quota, passphrase, alert, alert_delay)
+                          alert=None, alert_delay=None, creation=True):
+    """
+    Add a backup repository
+    """
+    args = {k: v for k, v in locals().items() if v is not None}
+    repository = BackupRepository(shortname, creation=True)
+    return repository.set(
+        operation_logger=args.pop('operation_logger')
+        args=urllib.parse.urlencode(args),
+    )
 
-
-def backup_repository_update(shortname, name=None,
+@is_unit_operation()
+def backup_repository_update(operation_logger, shortname, name=None,
                              quota=None, passphrase=None,
-                             alert=[], alert_delay=None):
-    return yunohost.repository.backup_repository_update(shortname, name, quota, passphrase, alert, alert_delay)
+                             alert=None, alert_delay=None):
+    """
+    Update a backup repository
+    """
 
+    backup_repository_add(creation=False, **locals()):
 
-def backup_repository_remove(shortname, purge=False):
-    return yunohost.repository.backup_repository_remove(shortname, purge)
-
+@is_unit_operation()
+def backup_repository_remove(operation_logger, shortname, purge=False):
+    """
+    Remove a backup repository
+    """
+    BackupRepository(shortname).remove(purge)
 
 
 #
-# End Repository subcategory
+# Timer subcategory
 #
 
+class BackupTimer(ConfigPanel):
+    """
+    BackupRepository manage all repository the admin added to the instance
+    """
+    entity_type = "backup_timer"
+    save_path_tpl = "/etc/systemd/system/backup_timer_{entity}.timer"
+    save_mode = "full"
+
+    @property
+    def service_path(self):
+        return self.save_path[:-len(".timer")] + ".service"
+
+    def _load_current_values(self):
+        # Inject defaults if needed (using the magic .update() ;))
+        self.values = self._get_default_values()
+
+        if os.path.exists(self.save_path) and os.path.isfile(self.save_path):
+            raise NotImplementedError() # TODO
+
+        if os.path.exists(self.service_path) and os.path.isfile(self.service_path):
+            raise NotImplementedError() # TODO
+
+
+    def _apply(self, values):
+        write_to_file(self.save_path, f"""[Unit]
+Description=Run backup {self.entity} regularly
+
+[Timer]
+OnCalendar={values['schedule']}
+
+[Install]
+WantedBy=timers.target
+""")
+        # TODO --system and --apps params
+        # TODO prune params
+        write_to_file(self.service_path, f"""[Unit]
+Description=Run backup {self.entity}
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/yunohost backup create -n '{name}' -r '{repo}' --system --apps ; /usr/bin/yunohost backup prune -n '{name}'
+User=root
+Group=root
+""")
+
+
+def backup_timer_list(full=False):
+    """
+    List all backup timer
+    """
+    return { "backup_timer": BackupTimer.list(full) }
+
+
+def backup_timer_info(shortname, space_used=False):
+    return BackupTimer(shortname).info(space_used)
+
+
+@is_unit_operation()
+def backup_timer_add(
+    operation_logger,
+    name=None,
+    description=None,
+    repos=[],
+    system=[],
+    apps=[],
+    schedule=None,
+    keep_hourly=None,
+    keep_daily=None,
+    keep_weekly=None,
+    keep_monthly=None,
+    creation=True,
+):
+    """
+    Add a backup timer
+    """
+    args = {k: v for k, v in locals().items() if v is not None}
+    timer = BackupTimer(shortname, creation=True)
+    return timer.set(
+        operation_logger=args.pop('operation_logger')
+        args=urllib.parse.urlencode(args),
+    )
+
+@is_unit_operation()
+def backup_timer_update(operation_logger, shortname, name=None,
+                             quota=None, passphrase=None,
+                             alert=None, alert_delay=None):
+    """
+    Update a backup timer
+    """
+
+    backup_timer_add(creation=False, **locals()):
+
+@is_unit_operation()
+def backup_timer_remove(operation_logger, shortname, purge=False):
+    """
+    Remove a backup timer
+    """
+    BackupTimer(shortname).remove(purge)
+
+
 
 #
 # Misc helpers                                                              #
diff --git a/src/yunohost/repositories/borg.py b/src/yunohost/repositories/borg.py
new file mode 100644
index 000000000..5ace1ec6d
--- /dev/null
+++ b/src/yunohost/repositories/borg.py
@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2013 Yunohost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+
+class BorgBackupRepository(LocalBackupRepository):
+    need_organized_files = True
+    method_name = "borg"
+
+    # TODO logs
+    def _run_borg_command(self, cmd, stdout=None):
+        """ Call a submethod of borg with the good context
+        """
+        env = dict(os.environ)
+
+        if self.domain:
+            # TODO Use the best/good key
+            private_key = "/root/.ssh/ssh_host_ed25519_key"
+
+            # Don't check ssh fingerprint strictly the first time
+            # TODO improve this by publishing and checking this with DNS
+            strict = 'yes' if self.domain in open('/root/.ssh/known_hosts').read() else 'no'
+            env['BORG_RSH'] = "ssh -i %s -oStrictHostKeyChecking=%s"
+            env['BORG_RSH'] = env['BORG_RSH'] % (private_key, strict)
+
+        # In case, borg need a passphrase to get access to the repo
+        if "passphrase" in self.future_values:
+            env['BORG_PASSPHRASE'] = self.passphrase
+
+        # Authorize to move the repository (borgbase do this)
+        env["BORG_RELOCATED_REPO_ACCESS_IS_OK"] = "yes"
+
+        return subprocess.Popen(cmd, env=env, stdout=stdout)
+
+    def _call(self, action, cmd, json_output=False):
+        borg = self._run_borg_command(cmd)
+        return_code = borg.wait()
+        if return_code:
+            raise YunohostError(f"backup_borg_{action}_error")
+
+        out, _ = borg.communicate()
+        if json_output:
+            try:
+                return json.loads(out)
+            except json.decoder.JSONDecodeError, TypeError:
+                raise YunohostError(f"backup_borg_{action}_error")
+        return out
+
+    # =================================================
+    # Repository actions
+    # =================================================
+
+    def install(self):
+        # Remote
+        if self.is_remote:
+            if self.is_shf and not self.future_values.get('user'):
+                services = {
+                    'borg': 'borgbackup'
+                }
+
+                response = shf_request(
+                    domain=self.domain,
+                    service=services[self.method],
+                    shf_id=values.pop('shf_id', None),
+                    data = {
+                        'origin': self.domain,
+                        'public_key': self.public_key,
+                        'quota':  self.quota,
+                        'alert':  self.alert,
+                        'alert_delay':  self.alert_delay,
+                        #password: "XXXXXXXX",
+                    }
+                )
+                self.new_values['shf_id'] = response['id']
+                self.new_values['location'] = response['repository']
+            elif not self.is_shf:
+                self.new_values['location'] = self.location
+
+            if not self.future_values.get('user'):
+                raise YunohostError("")
+        # Local
+        else:
+            super().install()
+
+
+        # Initialize borg repo
+        cmd = ["borg", "init", "--encryption", "repokey", self.location]
+
+        if "quota" in self.future_values:
+            cmd += ['--storage-quota', self.quota]
+        self._call('init', cmd)
+
+
+    def update(self):
+        raise NotImplementedError()
+
+    def purge(self):
+        if self.is_shf:
+            response = shf_request(
+                domain=self.domain,
+                service="borgbackup",
+                shf_id=values.pop('shf_id', None),
+                data = {
+                    'origin': self.domain,
+                    #password: "XXXXXXXX",
+                }
+            )
+        else:
+            cmd = ["borg", "delete", self.location]
+            self._call('purge', cmd)
+            if not self.is_remote:
+                super().purge()
+
+    def list_archives_names(self, prefix=None):
+        cmd = ["borg", "list", "--json", self.location]
+        if prefix:
+            cmd += ["-P", prefix]
+        response = self._call('list', cmd, True)
+        return [archive["name"] for archive in response['archives']]
+
+    def compute_space_used(self):
+        if not self.is_remote:
+            return super().purge()
+        else:
+            cmd = ["borg", "info", "--json", self.location]
+            response = self._call('info', cmd)
+            return response["cache"]["stats"]["unique_size"]
+
+    def prune(self, prefix=None, hourly=None, daily=None, weekly=None, yearly=None):
+
+        # List archives with creation date
+        archives = {}
+        for archive_name in self.list_archive_name(prefix):
+            archive = BackupArchive(repo=self, name=archive_name)
+            created_at = archive.info()["created_at"]
+            archives[created_at] = archive
+
+        if not archives:
+            return
+
+        # Generate period in which keep one archive
+        now = datetime.now()
+        periods = []
+        for in range(hourly):
+
+
+
+        for created_at in sorted(archives, reverse=True):
+            created_at = datetime.fromtimestamp(created_at)
+            if hourly > 0 and :
+                hourly-=1
+
+            archive.delete()
+
+
+class BorgBackupArchive(BackupArchive):
+        """ Backup prepared files with borg """
+
+    def backup(self):
+        cmd = ['borg', 'create', self.archive_path, './']
+        self.repo._call('backup', cmd)
+
+    def delete(self):
+        cmd = ['borg', 'delete', '--force', self.archive_path]
+        self.repo._call('delete_archive', cmd)
+
+    def list(self):
+        """ Return a list of archives names
+
+        Exceptions:
+        backup_borg_list_error -- Raised if the borg script failed
+        """
+        cmd = ["borg", "list", "--json-lines", self.archive_path]
+        out = self.repo._call('list_archive', cmd)
+        result = [json.loads(out) for line in out.splitlines()]
+        return result
+
+    def download(self, exclude_paths=[]):
+        super().download()
+        paths = self.select_files()
+        if isinstance(exclude_paths, str):
+            exclude_paths = [exclude_paths]
+        # Here tar archive are not compressed, if we want to compress we
+        # should add --tar-filter=gzip.
+        cmd = ["borg", "export-tar", self.archive_path, "-"] + paths
+        for path in exclude_paths:
+            cmd += ['--exclude', path]
+        reader = self.repo._run_borg_command(cmd, stdout=subprocess.PIPE)
+
+        # We return a raw bottle HTTPresponse (instead of serializable data like
+        # list/dict, ...), which is gonna be picked and used directly by moulinette
+        from bottle import response, HTTPResponse
+        response.content_type = "application/x-tar"
+        return HTTPResponse(reader, 200)
+
+    def extract(self, paths=None, exclude_paths=[]):
+        paths, exclude_paths = super().extract(paths, exclude_paths)
+        cmd = ['borg', 'extract', self.archive_path] + paths
+        for path in exclude_paths:
+            cmd += ['--exclude', path]
+        return self.repo._call('extract_archive', cmd)
+
+    def mount(self, path):
+        # FIXME How to be sure the place where we mount is secure ?
+        cmd = ['borg', 'mount', self.archive_path, path]
+        self.repo._call('mount_archive', cmd)
+
+
diff --git a/src/yunohost/repositories/hook.py b/src/yunohost/repositories/hook.py
new file mode 100644
index 000000000..817519162
--- /dev/null
+++ b/src/yunohost/repositories/hook.py
@@ -0,0 +1,128 @@
+
+class HookBackupRepository(BackupRepository):
+    method_name = "hook"
+
+    # =================================================
+    # Repository actions
+    # =================================================
+    def install(self):
+        raise NotImplementedError()
+
+    def update(self):
+        raise NotImplementedError()
+
+    def remove(self, purge=False):
+        if self.__class__ == BackupRepository:
+            raise NotImplementedError() # purge
+
+        rm(self.save_path, force=True)
+        logger.success(m18n.n("repository_removed", repository=self.shortname))
+
+    def list(self):
+        raise NotImplementedError()
+
+    def info(self, space_used=False):
+        result = super().get(mode="export")
+
+        if self.__class__ == BackupRepository and space_used == True:
+            raise NotImplementedError() # purge
+
+        return {self.shortname: result}
+
+    def prune(self):
+        raise NotImplementedError()
+
+
+class HookBackupArchive(BackupArchive):
+    # =================================================
+    # Archive actions
+    # =================================================
+    def backup(self):
+        raise NotImplementedError()
+        """
+        Launch a custom script to backup
+        """
+
+        self._call('backup', self.work_dir, self.name, self.repo.location, self.manager.size,
+                self.manager.description)
+
+    def restore(self):
+        raise NotImplementedError()
+
+    def delete(self):
+        raise NotImplementedError()
+
+    def list(self):
+        raise NotImplementedError()
+        """ Return a list of archives names
+
+        Exceptions:
+        backup_custom_list_error -- Raised if the custom script failed
+        """
+        out = self._call('list', self.repo.location)
+        result = out.strip().splitlines()
+        return result
+
+    def info(self):
+        raise NotImplementedError() #compute_space_used
+        """ Return json string of the info.json file
+
+        Exceptions:
+        backup_custom_info_error -- Raised if the custom script failed
+        """
+        return self._call('info', self.name, self.repo.location)
+
+    def download(self):
+        raise NotImplementedError()
+
+    def mount(self):
+        raise NotImplementedError()
+        """
+        Launch a custom script to mount the custom archive
+        """
+        super().mount()
+        self._call('mount', self.work_dir, self.name, self.repo.location, self.manager.size,
+                self.manager.description)
+
+    def extract(self):
+        raise NotImplementedError()
+
+    def need_organized_files(self):
+        """Call the backup_method hook to know if we need to organize files"""
+        if self._need_mount is not None:
+            return self._need_mount
+
+        try:
+            self._call('nedd_mount')
+        except YunohostError:
+            return False
+        return True
+    def _call(self, *args):
+        """ Call a submethod of backup method hook
+
+        Exceptions:
+        backup_custom_ACTION_error -- Raised if the custom script failed
+        """
+        ret = hook_callback("backup_method", [self.method],
+                            args=args)
+
+        ret_failed = [
+            hook
+            for hook, infos in ret.items()
+            if any(result["state"] == "failed" for result in infos.values())
+        ]
+        if ret_failed:
+            raise YunohostError("backup_custom_" + args[0] + "_error")
+
+        return ret["succeed"][self.method]["stdreturn"]
+
+    def _get_args(self, action):
+        """Return the arguments to give to the custom script"""
+        return [
+            action,
+            self.work_dir,
+            self.name,
+            self.repo,
+            self.manager.size,
+            self.manager.description,
+        ]
diff --git a/src/yunohost/repositories/tar.py b/src/yunohost/repositories/tar.py
new file mode 100644
index 000000000..ca47795aa
--- /dev/null
+++ b/src/yunohost/repositories/tar.py
@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2013 Yunohost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+
+class TarBackupRepository(LocalBackupRepository):
+    need_organized_files = False
+    method_name = "tar"
+    def list_archives_names(self):
+        # Get local archives sorted according to last modification time
+        # (we do a realpath() to resolve symlinks)
+        archives = glob(f"{self.location}/*.tar.gz") + glob(f"{self.location}/*.tar")
+        archives = set([os.path.realpath(archive) for archive in archives])
+        archives = sorted(archives, key=lambda x: os.path.getctime(x))
+
+        # Extract only filename without the extension
+        def remove_extension(f):
+            if f.endswith(".tar.gz"):
+                return os.path.basename(f)[: -len(".tar.gz")]
+            else:
+                return os.path.basename(f)[: -len(".tar")]
+
+        return [remove_extension(f) for f in archives]
+
+    def compute_space_used(self):
+        return space_used_in_directory(self.location)
+
+    def prune(self):
+        raise NotImplementedError()
+
+
+class TarBackupArchive:
+    @property
+    def archive_path(self):
+
+        if isinstance(self.manager, BackupManager) and settings_get(
+            "backup.compress_tar_archives"
+        ):
+            return os.path.join(self.repo.location, self.name + ".tar.gz")
+
+        f = os.path.join(self.repo.path, self.name + ".tar")
+        if os.path.exists(f + ".gz"):
+            f += ".gz"
+        return f
+
+    def backup(self):
+        # Open archive file for writing
+        try:
+            tar = tarfile.open(
+                self.archive_path,
+                "w:gz" if self.archive_path.endswith(".gz") else "w",
+            )
+        except Exception:
+            logger.debug(
+                "unable to open '%s' for writing", self.archive_path, exc_info=1
+            )
+            raise YunohostError("backup_archive_open_failed")
+
+        # Add files to the archive
+        try:
+            for path in self.manager.paths_to_backup:
+                # Add the "source" into the archive and transform the path into
+                # "dest"
+                tar.add(path["source"], arcname=path["dest"])
+        except IOError:
+            logger.error(
+                m18n.n(
+                    "backup_archive_writing_error",
+                    source=path["source"],
+                    archive=self._archive_file,
+                    dest=path["dest"],
+                ),
+                exc_info=1,
+            )
+            raise YunohostError("backup_creation_failed")
+        finally:
+            tar.close()
+
+        # Move info file
+        shutil.copy(
+            os.path.join(self.work_dir, "info.json"),
+            os.path.join(ARCHIVES_PATH, self.name + ".info.json"),
+        )
+
+        # If backuped to a non-default location, keep a symlink of the archive
+        # to that location
+        link = os.path.join(self.repo.path, self.name + ".tar")
+        if not os.path.isfile(link):
+            os.symlink(self.archive_path, link)
+
+    def copy(self, file, target):
+        tar = tarfile.open(
+            self._archive_file, "r:gz" if self._archive_file.endswith(".gz") else "r"
+        )
+        file_to_extract = tar.getmember(file)
+        # Remove the path
+        file_to_extract.name = os.path.basename(file_to_extract.name)
+        tar.extract(file_to_extract, path=target)
+        tar.close()
+
+    def delete(self):
+        archive_file = f"{self.repo.location}/{self.name}.tar"
+        info_file = f"{self.repo.location}/{self.name}.info.json"
+        if os.path.exists(archive_file + ".gz"):
+            archive_file += ".gz"
+
+        files_to_delete = [archive_file, info_file]
+
+        # To handle the case where archive_file is in fact a symlink
+        if os.path.islink(archive_file):
+            actual_archive = os.path.realpath(archive_file)
+            files_to_delete.append(actual_archive)
+
+        for backup_file in files_to_delete:
+            if not os.path.exists(backup_file):
+                continue
+            try:
+                os.remove(backup_file)
+            except Exception:
+                logger.debug("unable to delete '%s'", backup_file, exc_info=1)
+                logger.warning(m18n.n("backup_delete_error", path=backup_file))
+
+    def list(self):
+        try:
+            tar = tarfile.open(
+                self.archive_path,
+                "r:gz" if self.archive_path.endswith(".gz") else "r",
+            )
+        except Exception:
+            logger.debug(
+                "cannot open backup archive '%s'", self.archive_path, exc_info=1
+            )
+            raise YunohostError("backup_archive_open_failed")
+
+        try:
+            return tar.getnames()
+        except (IOError, EOFError, tarfile.ReadError) as e:
+            tar.close()
+            raise YunohostError(
+                "backup_archive_corrupted", archive=self.archive_path, error=str(e)
+            )
+
+    def download(self):
+        super().download()
+        # If symlink, retrieve the real path
+        archive_file = self.archive_path
+        if os.path.islink(archive_file):
+            archive_file = os.path.realpath(archive_file)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+                raise YunohostValidationError(
+                    "backup_archive_broken_link", path=archive_file
+                )
+
+        # We return a raw bottle HTTPresponse (instead of serializable data like
+        # list/dict, ...), which is gonna be picked and used directly by moulinette
+        from bottle import static_file
+
+        archive_folder, archive_file_name = archive_file.rsplit("/", 1)
+        return static_file(archive_file_name, archive_folder, download=archive_file_name)
+
+    def extract(self, paths=None, exclude_paths=[]):
+        paths, exclude_paths = super().extract(paths, exclude_paths)
+        # Mount the tarball
+        try:
+            tar = tarfile.open(
+                self.archive_path,
+                "r:gz" if self.archive_path.endswith(".gz") else "r",
+            )
+        except Exception:
+            logger.debug(
+                "cannot open backup archive '%s'", self.archive_path, exc_info=1
+            )
+            raise YunohostError("backup_archive_open_failed")
+
+        subdir_and_files = [
+            tarinfo
+            for tarinfo in tar.getmembers()
+            if (
+               any([tarinfo.name.startswith(path) for path in paths])
+               and all([not tarinfo.name.startswith(path) for path in exclude_paths])
+            )
+        ]
+        tar.extractall(members=subdir_and_files, path=self.work_dir)
+        tar.close()
+
+    def mount(self):
+        raise NotImplementedError()
+
+    def _archive_exists(self):
+        return os.path.lexists(self.archive_path)
+
+    def _assert_archive_exists(self):
+        if not self._archive_exists():
+            raise YunohostError('backup_archive_name_unknown', name=self.name)
+
+        # If symlink, retrieve the real path
+        if os.path.islink(self.archive_path):
+            archive_file = os.path.realpath(self.archive_path)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+                raise YunohostError('backup_archive_broken_link',
+                                    path=archive_file)
+
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index ab5e6d62b..8a540867b 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -32,246 +32,648 @@ import urllib.parse
 
 from moulinette import Moulinette, m18n
 from moulinette.core import MoulinetteError
-from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, read_yaml, write_to_json
+from moulinette.utils.filesystem import read_file, read_yaml, write_to_json, rm, mkdir, chmod, chown
+from moulinette.utils.network import download_text, download_json
 
 
 from yunohost.utils.config import ConfigPanel, Question
 from yunohost.utils.error import YunohostError
-from yunohost.utils.filesystem import binary_to_human
-from yunohost.utils.network import get_ssh_public_key
+from yunohost.utils.filesystem import space_used_in_directory, disk_usage, binary_to_human
+from yunohost.utils.network import get_ssh_public_key, shf_request, SHF_BASE_URL
 from yunohost.log import OperationLogger, is_unit_operation
 
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_DIR = '/etc/yunohost/repositories'
+CACHE_INFO_DIR = "/var/cache/yunohost/{repository}"
 REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
-
-# TODO i18n
-# TODO split COnfigPanel.get to extract "Format result" part and be able to override it
+# TODO split ConfigPanel.get to extract "Format result" part and be able to override it
 # TODO Migration
 # TODO Remove BackupRepository.get_or_create()
 # TODO Backup method
-# TODO auto test F2F by testing .well-known url
 # TODO API params to get description of forms
 # TODO tests
 # TODO detect external hard drive already mounted and suggest it
-# TODO F2F client detection / add / update / delete
+# TODO F2F client delete
 # TODO F2F server
+# TODO i18n pattern error
 
 class BackupRepository(ConfigPanel):
     """
     BackupRepository manage all repository the admin added to the instance
     """
-    @classmethod
-    def get(cls, shortname):
-        # FIXME
-        if name not in cls.repositories:
-            raise YunohostError('backup_repository_doesnt_exists', name=name)
+    entity_type = "backup_repository"
+    save_path_tpl = "/etc/yunohost/backup/repositories/{entity}.yml"
+    save_mode = "full"
+    need_organized_files = True
+    method_name = ""
 
-        return cls.repositories[name]
-
-    def __init__(self, repository):
-        self.repository = repository
-        self.save_mode = "full"
-        super().__init__(
-            config_path=REPOSITORY_CONFIG_PATH,
-            save_path=f"{REPOSITORIES_DIR}/{repository}.yml",
-        )
-
-        #self.method = BackupMethod.get(method, self)
-
-    def set__domain(self, question):
-        # TODO query on domain name .well-known
-        question.value
-
-    def _get_default_values(self):
-        values = super()._get_default_values()
-        values["public_key"] = get_ssh_public_key()
-        return values
-
-    def list(self, with_info=False):
-        return self.method.list(with_info)
-
-    def compute_space_used(self):
-        if self.used is None:
-            try:
-                self.used = self.method.compute_space_used()
-            except (AttributeError, NotImplementedError):
-                self.used = 'unknown'
-        return self.used
-
-    def purge(self):
-        # TODO F2F delete
-        self.method.purge()
-
-    def delete(self, purge=False):
-
-        if purge:
-            self.purge()
-
-        os.system("rm -rf {REPOSITORY_SETTINGS_DIR}/{self.repository}.yml")
-
-
-    def _split_location(self):
+    @staticmethod
+    def split_location(location):
         """
         Split a repository location into protocol, user, domain and path
         """
-        location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+:))?(?P<path>[^\0]+)$'
-        location_match = re.match(location_regex, self.location)
+        if "/" not in location:
+            return { "domain": location }
+
+        location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+):((?P<port>\d+)/)?)?(?P<path>[^:]+)$'
+        location_match = re.match(location_regex, location)
 
         if location_match is None:
             raise YunohostError('backup_repositories_invalid_location',
                                 location=location)
+        return {
+            'protocol': location_match.group('protocol'),
+            'user': location_match.group('user'),
+            'domain': location_match.group('domain'),
+            'port': location_match.group('port'),
+            'path': location_match.group('path')
+        }
 
-        self.protocol = location_match.group('protocol')
-        self.user = location_match.group('user')
-        self.domain = location_match.group('domain')
-        self.path = location_match.group('path')
+    @classmethod
+    def list(cls, space_used=False, full=False):
+        """
+        List available repositories where put archives
+        """
+        repositories = super().list()
 
+        if not full:
+            return repositories
 
-def backup_repository_list(full=False):
-    """
-    List available repositories where put archives
-    """
+        for repo in repositories:
+            try:
+                repositories[repo] = BackupRepository(repo).info(space_used)
+            except Exception as e:
+                logger.error(f"Unable to open repository {repo}")
 
-    try:
-        repositories = [f.rstrip(".yml")
-                        for f in os.listdir(REPOSITORIES_DIR)
-                        if os.path.isfile(f) and f.endswith(".yml")]
-    except FileNotFoundError:
-        repositories = []
-
-    if not full:
         return repositories
 
-    # FIXME: what if one repo.yml is corrupted ?
-    repositories = {repo: BackupRepository(repo).get(mode="export")
-                    for repo in repositories}
 
-    return repositories
+    # =================================================
+    # Config Panel Hooks
+    # =================================================
+
+    def post_ask__domain(self, question):
+        """ Detect if the domain support Self-Hosting Federation protocol
+        """
+        #import requests
+        # FIXME What if remote server is self-signed ?
+        # FIXME What if remote server is unreachable temporarily ?
+        url = SHF_BASE_URL.format(domain=question.value) + "/"
+        try:
+            #r = requests.get(url, timeout=10)
+            download_text(url, timeout=10)
+        except MoulinetteError as e:
+            logger.debug("SHF not running")
+            return { 'is_shf': False }
+        logger.debug("SHF running")
+        return { 'is_shf': True }
 
 
-def backup_repository_info(shortname, human_readable=True, space_used=False):
-    """
-    Show info about a repository
+    # =================================================
+    # Config Panel Override
+    # =================================================
+    def _get_default_values(self):
+        values = super()._get_default_values()
+        # TODO move that in a getter hooks ?
+        values["public_key"] = get_ssh_public_key()
+        return values
 
-    Keyword arguments:
-        name -- Name of the backup repository
-    """
-    Question.operation_logger = operation_logger
-    repository = BackupRepository(shortname)
-    # TODO
-    if space_used:
-        repository.compute_space_used()
+    def _load_current_values(self):
+        super()._load_current_values()
 
-    repository = repository.get(
-        mode="export"
-    )
+        if 'location' in self.values:
+            self.values.update(BackupRepository.split_location(self.values['location']))
+        self.values['is_remote'] = bool(self.values.get('domain'))
 
-    if human_readable:
-        if 'quota' in repository:
-            repository['quota'] = binary_to_human(repository['quota'])
-        if 'used' in repository and isinstance(repository['used'], int):
-            repository['used'] = binary_to_human(repository['used'])
+        if self.values.get('method') == 'tar' and self.values['is_remote']:
+            raise YunohostError("repository_tar_only_local")
 
-    return repository
+        if 'shf_id' in self.values:
+            self.values['is_shf'] = bool(self.values['shf_id'])
+        self._cast_by_method()
 
+    def _parse_pre_answered(self, *args):
+        super()._parse_pre_answered(*args)
+        if 'location'  in self.args:
+            self.args.update(BackupRepository.split_location(self.args['location']))
+        if 'domain' in self.args:
+            self.args['is_remote'] = bool(self.args['domain'])
+            self.args['method'] = "borg"
+        elif self.args.get('method') == 'tar':
+            self.args['is_remote'] = False
+        self._cast_by_method()
 
-@is_unit_operation()
-def backup_repository_add(operation_logger, shortname, name=None, location=None,
-                          method=None, quota=None, passphrase=None,
-                          alert=[], alert_delay=7):
-    """
-    Add a backup repository
-
-    Keyword arguments:
-        location -- Location of the repository (could be a remote location)
-        shortname -- Name of the backup repository
-        name -- An optionnal description
-        quota -- Maximum size quota of the repository
-        encryption -- If available, the kind of encryption to use
-    """
-    # FIXME i18n
-    # Deduce some value from location
-    args = {}
-    args['description'] = name
-    args['creation'] = True
-    if location:
-        args["location"] = location
-        args["is_remote"] = True
-        args["method"] = method if method else "borg"
-        domain_re = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        if re.match(domain_re, location):
-            args["is_f2f"] = True
-        elif location[0] != "/":
-            args["is_f2f"] = False
+    def _apply(self):
+        # Activate / update services
+        if not os.path.exists(self.save_path):
+            self.install()
         else:
-            args["is_remote"] = False
-            args["method"] = method
-    elif method == "tar":
-        args["is_remote"] = False
-    if not location:
-        args["method"] = method
+            self.update()
 
-    args["quota"] = quota
-    args["passphrase"] = passphrase
-    args["alert"]= ",".join(alert) if alert else None
-    args["alert_delay"]= alert_delay
-
-    # TODO validation
-    # TODO activate service in apply (F2F or not)
-    Question.operation_logger = operation_logger
-    repository = BackupRepository(shortname)
-    return repository.set(
-        args=urllib.parse.urlencode(args),
-        operation_logger=operation_logger
-    )
+        # Clean redundant values before to register
+        for prop in ['is_remote', 'domain', 'port', 'user', 'path',
+                     'creation', 'is_shf', 'shortname']:
+            self.values.pop(prop, None)
+            self.new_values.pop(prop, None)
+        super()._apply()
 
 
-@is_unit_operation()
-def backup_repository_update(operation_logger, shortname, name=None,
-                             quota=None, passphrase=None,
-                             alert=[], alert_delay=None):
-    """
-    Update a backup repository
+    # =================================================
+    # BackupMethod encapsulation
+    # =================================================
+    @property
+    def location(self):
+        if not self.future_values:
+            return None
 
-    Keyword arguments:
-        name -- Name of the backup repository
-    """
+        if not self.is_remote:
+            return self.path
+
+        return f"ssh://{self.user}@{self.domain}:{self.port}/{self.path}"
+
+    def _cast_by_method(self):
+        if not self.future_values:
+            return
+
+        if self.__class__ == BackupRepository:
+            if self.method == 'tar':
+                self.__class__ = TarBackupRepository
+            elif self.method == 'borg':
+                self.__class__ = BorgBackupRepository
+            else:
+                self.__class__ = HookBackupRepository
+
+    def _check_is_enough_free_space(self):
+        """
+        Check free space in repository or output directory before to backup
+        """
+        # TODO How to do with distant repo or with deduplicated backup ?
+        backup_size = self.manager.size
+
+        free_space = free_space_in_directory(self.repo)
+
+        if free_space < backup_size:
+            logger.debug(
+                "Not enough space at %s (free: %s / needed: %d)",
+                self.repo,
+                free_space,
+                backup_size,
+            )
+            raise YunohostValidationError("not_enough_disk_space", path=self.repo)
+
+    def remove(self, purge=False):
+        if purge:
+            self._load_current_values()
+            self.purge()
+
+        rm(self.save_path, force=True)
+        logger.success(m18n.n("repository_removed", repository=self.shortname))
+
+    def info(self, space_used=False):
+        result = super().get(mode="export")
+
+        if self.__class__ == BackupRepository and space_used == True:
+            result["space_used"] = self.compute_space_used()
+
+        return {self.shortname: result}
+
+    def list(self, with_info):
+        archives = self.list_archive_name()
+        if with_info:
+            d = OrderedDict()
+            for archive in archives:
+                try:
+                    d[archive] = BackupArchive(repo=self, name=archive).info()
+                except YunohostError as e:
+                    logger.warning(str(e))
+                except Exception:
+                    import traceback
+
+                    logger.warning(
+                        "Could not check infos for archive %s: %s"
+                        % (archive, "\n" + traceback.format_exc())
+                    )
+
+            archives = d
+
+        return archives
+
+    # =================================================
+    # Repository abstract actions
+    # =================================================
+    def install(self):
+        raise NotImplementedError()
+
+    def update(self):
+        raise NotImplementedError()
+
+    def purge(self):
+        raise NotImplementedError()
+
+    def list_archives_names(self):
+        raise NotImplementedError()
+
+    def compute_space_used(self):
+        raise NotImplementedError()
+
+    def prune(self):
+        raise NotImplementedError() # TODO prune
+
+class LocalBackupRepository(BackupRepository):
+    def install(self):
+        self.new_values['location'] = self.location
+        mkdir(self.location, mode=0o0750, parents=True, uid="admin", gid="root", force=True)
+
+    def update(self):
+        self.install()
+
+    def purge(self):
+        rm(self.location, recursive=True, force=True)
+
+
+class BackupArchive:
+    def __init__(self, repo, name=None, manager=None):
+        self.manager = manager
+        self.name = name or manager.name
+        if self.name.endswith(".tar.gz"):
+            self.name = self.name[: -len(".tar.gz")]
+        elif self.name.endswith(".tar"):
+            self.name = self.name[: -len(".tar")]
+        self.repo = repo
+
+        # Cast
+        if self.repo.method_name == 'tar':
+            self.__class__ = TarBackupArchive
+        elif self.repo.method_name == 'borg':
+            self.__class__ = BorgBackupArchive
+        else:
+            self.__class__ = HookBackupArchive
+
+        # Assert archive exists
+        if not isinstance(self.manager, BackupManager) and self.name not in self.repo.list():
+            raise YunohostValidationError("backup_archive_name_unknown", name=name)
+
+    @property
+    def archive_path(self):
+        """Return the archive path"""
+        return self.repo.location + '::' + self.name
+
+    @property
+    def work_dir(self):
+        """
+        Return the working directory
+
+        For a BackupManager, it is the directory where we prepare the files to
+        backup
+
+        For a RestoreManager, it is the directory where we mount the archive
+        before restoring
+        """
+        return self.manager.work_dir
+
+    # This is not a property cause it could be managed in a hook
+    def need_organized_files(self):
+        return self.repo.need_organised_files
+
+    def organize_and_backup(self):
+        """
+        Run the backup on files listed by  the BackupManager instance
+
+        This method shouldn't be overrided, prefer overriding self.backup() and
+        self.clean()
+        """
+        if self.need_organized_files():
+            self._organize_files()
+
+        self.repo.install()
+
+        # Check free space in output
+        self._check_is_enough_free_space()
+        try:
+            self.backup()
+        finally:
+            self.clean()
+
+    def select_files(self):
+        files_in_archive = self.list()
+
+        if "info.json" in files_in_archive:
+            leading_dot = ""
+            yield "info.json"
+        elif "./info.json" in files_in_archive:
+            leading_dot = "./"
+            yield "./info.json"
+        else:
+            logger.debug(
+                "unable to retrieve 'info.json' inside the archive", exc_info=1
+            )
+            raise YunohostError(
+                "backup_archive_cant_retrieve_info_json", archive=self.archive_path
+            )
+        extract_paths = []
+        if f"{leading_dot}backup.csv" in files_in_archive:
+            yield f"{leading_dot}backup.csv"
+        else:
+            # Old backup archive have no backup.csv file
+            pass
+
+        # Extract system parts backup
+        conf_extracted = False
+
+        system_targets = self.manager.targets.list("system", exclude=["Skipped"])
+        apps_targets = self.manager.targets.list("apps", exclude=["Skipped"])
+
+        for system_part in system_targets:
+            if system_part.startswith("conf_"):
+                if conf_extracted:
+                    continue
+                system_part = "conf/"
+                conf_extracted = True
+            else:
+                system_part = system_part.replace("_", "/") + "/"
+            yield leading_dot + system_part
+        yield f"{leading_dot}hook/restore/"
+
+        # Extract apps backup
+        for app in apps_targets:
+            yield f"{leading_dot}apps/{app}"
+
+    def _get_info_string(self):
+        archive_file = "%s/%s.tar" % (self.repo.path, self.name)
+
+        # Check file exist (even if it's a broken symlink)
+        if not os.path.lexists(archive_file):
+            archive_file += ".gz"
+            if not os.path.lexists(archive_file):
+                raise YunohostValidationError("backup_archive_name_unknown", name=name)
+
+        # If symlink, retrieve the real path
+        if os.path.islink(archive_file):
+            archive_file = os.path.realpath(archive_file)
+
+            # Raise exception if link is broken (e.g. on unmounted external storage)
+            if not os.path.exists(archive_file):
+                raise YunohostValidationError(
+                    "backup_archive_broken_link", path=archive_file
+                )
+        info_file = CACHE_INFO_DIR.format(repository=self.repo.name)
+        mkdir(info_file, mode=0o0700, parents=True, force=True)
+        info_file += f"/{self.name}.info.json"
+
+        if not os.path.exists(info_file):
+            info_dir = tempfile.mkdtemp()
+            try:
+                files_in_archive = self.list()
+                if "info.json" in files_in_archive:
+                    self.extract("info.json")
+                elif "./info.json" in files_in_archive:
+                    self.extract("./info.json")
+                else:
+                    raise YunohostError(
+                        "backup_archive_cant_retrieve_info_json", archive=archive_file
+                    )
+                shutil.move(os.path.join(info_dir, "info.json"), info_file)
+            finally:
+                os.rmdir(info_dir)
+
+        try:
+            return read_file(info_file)
+        except MoulinetteError:
+            logger.debug("unable to load '%s'", info_file, exc_info=1)
+            raise YunohostError('backup_invalid_archive')
+
+    def info(self):
+
+        info_json = self._get_info_string()
+        if not self._info_json:
+            raise YunohostError('backup_info_json_not_implemented')
+        try:
+            info = json.load(info_json)
+        except:
+            logger.debug("unable to load info json", exc_info=1)
+            raise YunohostError('backup_invalid_archive')
+
+        # (legacy) Retrieve backup size
+        # FIXME
+        size = info.get("size", 0)
+        if not size:
+            tar = tarfile.open(
+                archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
+            )
+            size = reduce(
+                lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
+            )
+            tar.close()
+        result = {
+            "path": repo.archive_path,
+            "created_at": datetime.utcfromtimestamp(info["created_at"]),
+            "description": info["description"],
+            "size": size,
+        }
+        if human_readable:
+            result['size'] = binary_to_human(result['size']) + 'B'
+
+        if with_details:
+            system_key = "system"
+            # Historically 'system' was 'hooks'
+            if "hooks" in info.keys():
+                system_key = "hooks"
+
+            if "size_details" in info.keys():
+                for category in ["apps", "system"]:
+                    for name, key_info in info[category].items():
+
+                        if category == "system":
+                            # Stupid legacy fix for weird format between 3.5 and 3.6
+                            if isinstance(key_info, dict):
+                                key_info = key_info.keys()
+                            info[category][name] = key_info = {"paths": key_info}
+                        else:
+                            info[category][name] = key_info
+
+                        if name in info["size_details"][category].keys():
+                            key_info["size"] = info["size_details"][category][name]
+                            if human_readable:
+                                key_info["size"] = binary_to_human(key_info["size"]) + "B"
+                        else:
+                            key_info["size"] = -1
+                            if human_readable:
+                                key_info["size"] = "?"
+
+            result["apps"] = info["apps"]
+            result["system"] = info[system_key]
+            result["from_yunohost_version"] = info.get("from_yunohost_version")
+
+        return info
+
+# TODO move this in BackupManager ?????
+    def clean(self):
+        """
+        Umount sub directories of working dirextories and delete it if temporary
+        """
+        if self.need_organized_files():
+            if not _recursive_umount(self.work_dir):
+                raise YunohostError("backup_cleaning_failed")
+
+        if self.manager.is_tmp_work_dir:
+            filesystem.rm(self.work_dir, True, True)
+    def _organize_files(self):
+        """
+        Mount all csv src in their related path
+
+        The goal is to organize the files app by app and hook by hook, before
+        custom backup method or before the restore operation (in the case of an
+        unorganize archive).
+
+        The usage of binding could be strange for a user because the du -sb
+        command will return that the working directory is big.
+        """
+        paths_needed_to_be_copied = []
+        for path in self.manager.paths_to_backup:
+            src = path["source"]
+
+            if self.manager is RestoreManager:
+                # TODO Support to run this before a restore (and not only before
+                # backup). To do that RestoreManager.unorganized_work_dir should
+                # be implemented
+                src = os.path.join(self.unorganized_work_dir, src)
+
+            dest = os.path.join(self.work_dir, path["dest"])
+            if dest == src:
+                continue
+            dest_dir = os.path.dirname(dest)
+
+            # Be sure the parent dir of destination exists
+            if not os.path.isdir(dest_dir):
+                filesystem.mkdir(dest_dir, parents=True)
+
+            # For directory, attempt to mount bind
+            if os.path.isdir(src):
+                filesystem.mkdir(dest, parents=True, force=True)
+
+                try:
+                    subprocess.check_call(["mount", "--rbind", src, dest])
+                    subprocess.check_call(["mount", "-o", "remount,ro,bind", dest])
+                except Exception:
+                    logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest))
+                    # To check if dest is mounted, use /proc/mounts that
+                    # escape spaces as \040
+                    raw_mounts = read_file("/proc/mounts").strip().split("\n")
+                    mounts = [m.split()[1] for m in raw_mounts]
+                    mounts = [m.replace("\\040", " ") for m in mounts]
+                    if dest in mounts:
+                        subprocess.check_call(["umount", "-R", dest])
+                else:
+                    # Success, go to next file to organize
+                    continue
+
+            # For files, create a hardlink
+            elif os.path.isfile(src) or os.path.islink(src):
+                # Can create a hard link only if files are on the same fs
+                # (i.e. we can't if it's on a different fs)
+                if os.stat(src).st_dev == os.stat(dest_dir).st_dev:
+                    # Don't hardlink /etc/cron.d files to avoid cron bug
+                    # 'NUMBER OF HARD LINKS > 1' see #1043
+                    cron_path = os.path.abspath("/etc/cron") + "."
+                    if not os.path.abspath(src).startswith(cron_path):
+                        try:
+                            os.link(src, dest)
+                        except Exception as e:
+                            # This kind of situation may happen when src and dest are on different
+                            # logical volume ... even though the st_dev check previously match...
+                            # E.g. this happens when running an encrypted hard drive
+                            # where everything is mapped to /dev/mapper/some-stuff
+                            # yet there are different devices behind it or idk ...
+                            logger.warning(
+                                "Could not link %s to %s (%s) ... falling back to regular copy."
+                                % (src, dest, str(e))
+                            )
+                        else:
+                            # Success, go to next file to organize
+                            continue
+
+            # If mountbind or hardlink couldnt be created,
+            # prepare a list of files that need to be copied
+            paths_needed_to_be_copied.append(path)
+
+        if len(paths_needed_to_be_copied) == 0:
+            return
+        # Manage the case where we are not able to use mount bind abilities
+        # It could be just for some small files on different filesystems or due
+        # to mounting error
+
+        # Compute size to copy
+        size = sum(disk_usage(path["source"]) for path in paths_needed_to_be_copied)
+        size /= 1024 * 1024  # Convert bytes to megabytes
+
+        # Ask confirmation for copying
+        if size > MB_ALLOWED_TO_ORGANIZE:
+            try:
+                i = Moulinette.prompt(
+                    m18n.n(
+                        "backup_ask_for_copying_if_needed",
+                        answers="y/N",
+                        size=str(size),
+                    )
+                )
+            except NotImplemented:
+                raise YunohostError("backup_unable_to_organize_files")
+            else:
+                if i != "y" and i != "Y":
+                    raise YunohostError("backup_unable_to_organize_files")
+
+        # Copy unbinded path
+        logger.debug(m18n.n("backup_copying_to_organize_the_archive", size=str(size)))
+        for path in paths_needed_to_be_copied:
+            dest = os.path.join(self.work_dir, path["dest"])
+            if os.path.isdir(path["source"]):
+                shutil.copytree(path["source"], dest, symlinks=True)
+            else:
+                shutil.copy(path["source"], dest)
+
+    # =================================================
+    # Archive abstract actions
+    # =================================================
+    def backup(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+
+    def delete(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+
+    def list(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+
+
+    def download(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+        if Moulinette.interface.type != "api":
+            logger.error(
+                "This option is only meant for the API/webadmin and doesn't make sense for the command line."
+            )
+            return
+
+    def extract(self, paths=None, exclude_paths=[]):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
+        if isinstance(exclude_paths, str):
+            paths = [paths]
+        elif paths is None:
+            paths = self.select_files()
+        if isinstance(exclude_paths, str):
+            exclude_paths = [exclude_paths]
+        return paths, exclude_paths
+
+    def mount(self):
+        if self.__class__ == BackupArchive:
+            raise NotImplementedError()
 
-    args = {}
-    args['creation'] = False
-    if name:
-        args['description'] = name
-    if quota:
-        args["quota"] = quota
-    if passphrase:
-        args["passphrase"] = passphrase
-    if alert is not None:
-        args["alert"]= ",".join(alert) if alert else None
-    if alert_delay:
-        args["alert_delay"]= alert_delay
 
-    # TODO validation
-    # TODO activate service in apply
-    Question.operation_logger = operation_logger
-    repository = BackupRepository(shortname)
-    return repository.set(
-        args=urllib.parse.urlencode(args),
-        operation_logger=operation_logger
-    )
 
 
-@is_unit_operation()
-def backup_repository_remove(operation_logger, shortname, purge=False):
-    """
-    Remove a backup repository
 
-    Keyword arguments:
-        name -- Name of the backup repository to remove
 
-    """
-    BackupRepository(shortname).delete(purge)
-    logger.success(m18n.n('backup_repository_removed', repository=shortname,
-                        path=repository['path']))
diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py
index 3d3045e6f..dc5ae545a 100644
--- a/src/yunohost/utils/network.py
+++ b/src/yunohost/utils/network.py
@@ -29,6 +29,7 @@ from moulinette.utils.process import check_output
 
 logger = logging.getLogger("yunohost.utils.network")
 
+SHF_BASE_URL = "https://{domain}/.well-known/self-hosting-federation/v1"
 
 def get_public_ip(protocol=4):
 
@@ -167,6 +168,9 @@ def _extract_inet(string, skip_netmask=False, skip_loopback=True):
     return result
 
 def get_ssh_public_key():
+    """ Return the prefered public key
+    This is used by the Self-Hosting Federation protocol
+    """
     keys = [
         '/etc/ssh/ssh_host_ed25519_key.pub',
         '/etc/ssh/ssh_host_rsa_key.pub'
@@ -176,3 +180,46 @@ def get_ssh_public_key():
             # We return the key without user and machine name.
             # Providers don't need this info.
             return " ".join(read_file(key).split(" ")[0:2])
+
+def shf_request(domain, service, shf_id=None, data={}):
+    # Get missing info from SHF protocol
+    import requests
+    # We try to get the URL repo through SHFi
+    base_url = SHF_BASE_URL.format(domain=domain)
+    url = f"{base_url}/service/{service}"
+
+    # FIXME add signature mechanism and portection against replay attack
+    # FIXME add password to manage the service ?
+    # FIXME support self-signed destination domain by asking validation to user
+    try:
+        if data is None:
+            r = requests.delete(url, timeout=30)
+        else:
+            if shf_id:
+                r = requests.put(f"{url}/{shf_id}", data=data, timeout=30)
+            else:
+                r = requests.post(url, data=data, timeout=30)
+    # SSL exceptions
+    except requests.exceptions.SSLError:
+        raise MoulinetteError("download_ssl_error", url=url)
+    # Invalid URL
+    except requests.exceptions.ConnectionError:
+        raise MoulinetteError("invalid_url", url=url)
+    # Timeout exceptions
+    except requests.exceptions.Timeout:
+        raise MoulinetteError("download_timeout", url=url)
+    # Unknown stuff
+    except Exception as e:
+        raise MoulinetteError("download_unknown_error", url=url, error=str(e))
+    if r.status_code in [401, 403]:
+        if self.creation:
+            raise YunohostError("repository_shf_creation_{r.status_code}")
+        else:
+            response = r.json()
+            raise YunohostError("repository_shf_update_{r.status_code}", message=response['message'])
+
+    elif r.status_code in [200, 201, 202]:
+        return r.json()
+        # FIXME validate repository and id
+    else:
+        raise YunohostError("repository_shf_invalid")

From 00f8e95de97ee54caf029ad510835815c3dea436 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Tue, 26 Oct 2021 18:04:08 +0200
Subject: [PATCH 27/48] [wip] Repo pruning

---
 data/actionsmap/yunohost.yml          | 14 +++++--
 src/yunohost/repositories/__init__.py |  0
 src/yunohost/repositories/borg.py     | 57 +++++++++++++++++++--------
 src/yunohost/repositories/tar.py      | 17 +++++++-
 src/yunohost/repository.py            | 47 ++++++++++++++++++++--
 5 files changed, 110 insertions(+), 25 deletions(-)
 create mode 100644 src/yunohost/repositories/__init__.py

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index d3633bfac..4f3f0492d 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -1041,7 +1041,7 @@ backup:
                     help: Name of the backup archive
                     extra:
                         pattern: &pattern_backup_archive_name
-                            - !!str ^[\w\-\._]{1,50}(?<!\.)$
+                            - !!str ^[\w\-\._]{1,50}$
                             - "pattern_backup_archive_name"
                 -d:
                     full: --description
@@ -1255,9 +1255,9 @@ backup:
                         name:
                             help: Short prefix of the backup archives
                             extra:
-                                pattern: &pattern_backup_archive_name
-                                    - !!str ^[\w\-\._]{1,50}(?<!\.)$
-                                    - "pattern_backup_archive_name"
+                                pattern: &pattern_backup_timer_name
+                                    - !!str ^[\w\-\._]{1,50}$
+                                    - "pattern_backup_timer_name"
                         -d:
                             full: --description
                             help: Short description of the backup
@@ -1291,6 +1291,8 @@ backup:
                     arguments:
                         name:
                             help: Short prefix of the backup archives
+                            extra:
+                                pattern: *pattern_backup_timer_name
                         -d:
                             full: --description
                             help: Short description of the backup
@@ -1324,6 +1326,8 @@ backup:
                     arguments:
                         name:
                             help: Short prefix of the backup archives
+                            extra:
+                                pattern: *pattern_backup_timer_name
 
                 ### backup_timer_info()
                 info:
@@ -1332,6 +1336,8 @@ backup:
                     arguments:
                         name:
                             help: Short prefix of the backup archives
+                            extra:
+                                pattern: *pattern_backup_timer_name
 
 #############################
 #         Settings          #
diff --git a/src/yunohost/repositories/__init__.py b/src/yunohost/repositories/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/yunohost/repositories/borg.py b/src/yunohost/repositories/borg.py
index 5ace1ec6d..a9b1d4654 100644
--- a/src/yunohost/repositories/borg.py
+++ b/src/yunohost/repositories/borg.py
@@ -18,6 +18,18 @@
     along with this program; if not, see http://www.gnu.org/licenses
 
 """
+import os
+import subprocess
+import json
+
+from datetime import datetime
+
+from moulinette.utils.log import getActionLogger
+from moulinette import m18n
+
+from yunohost.utils.error import YunohostError
+from yunohost.repository import LocalBackupRepository
+logger = getActionLogger("yunohost.repository")
 
 
 class BorgBackupRepository(LocalBackupRepository):
@@ -59,7 +71,7 @@ class BorgBackupRepository(LocalBackupRepository):
         if json_output:
             try:
                 return json.loads(out)
-            except json.decoder.JSONDecodeError, TypeError:
+            except (json.decoder.JSONDecodeError, TypeError):
                 raise YunohostError(f"backup_borg_{action}_error")
         return out
 
@@ -79,12 +91,12 @@ class BorgBackupRepository(LocalBackupRepository):
                     domain=self.domain,
                     service=services[self.method],
                     shf_id=values.pop('shf_id', None),
-                    data = {
+                    data={
                         'origin': self.domain,
                         'public_key': self.public_key,
-                        'quota':  self.quota,
-                        'alert':  self.alert,
-                        'alert_delay':  self.alert_delay,
+                        'quota': self.quota,
+                        'alert': self.alert,
+                        'alert_delay': self.alert_delay,
                         #password: "XXXXXXXX",
                     }
                 )
@@ -99,7 +111,6 @@ class BorgBackupRepository(LocalBackupRepository):
         else:
             super().install()
 
-
         # Initialize borg repo
         cmd = ["borg", "init", "--encryption", "repokey", self.location]
 
@@ -107,7 +118,6 @@ class BorgBackupRepository(LocalBackupRepository):
             cmd += ['--storage-quota', self.quota]
         self._call('init', cmd)
 
-
     def update(self):
         raise NotImplementedError()
 
@@ -117,7 +127,7 @@ class BorgBackupRepository(LocalBackupRepository):
                 domain=self.domain,
                 service="borgbackup",
                 shf_id=values.pop('shf_id', None),
-                data = {
+                data={
                     'origin': self.domain,
                     #password: "XXXXXXXX",
                 }
@@ -143,7 +153,7 @@ class BorgBackupRepository(LocalBackupRepository):
             response = self._call('info', cmd)
             return response["cache"]["stats"]["unique_size"]
 
-    def prune(self, prefix=None, hourly=None, daily=None, weekly=None, yearly=None):
+    def prune(self, prefix=None, **kwargs):
 
         # List archives with creation date
         archives = {}
@@ -155,17 +165,30 @@ class BorgBackupRepository(LocalBackupRepository):
         if not archives:
             return
 
-        # Generate period in which keep one archive
-        now = datetime.now()
-        periods = []
-        for in range(hourly):
-
+        # Generate periods in which keep one archive
+        now = datetime.utcnow()
+        now -= timedelta(
+            minutes=now.minute,
+            seconds=now.second,
+            microseconds=now.microsecond
+        )
+        periods = set([])
 
+        for unit, qty in kwargs:
+            if not qty:
+                continue
+            period = timedelta(**{unit: 1})
+            periods += set([(now - period * i, now - period * (i - 1))
+                        for i in range(qty)])
 
+        # Delete unneeded archive
         for created_at in sorted(archives, reverse=True):
-            created_at = datetime.fromtimestamp(created_at)
-            if hourly > 0 and :
-                hourly-=1
+            created_at = datetime.utcfromtimestamp(created_at)
+            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
+
+            if keep_for:
+                periods -= keep_for
+                continue
 
             archive.delete()
 
diff --git a/src/yunohost/repositories/tar.py b/src/yunohost/repositories/tar.py
index ca47795aa..d49643ed7 100644
--- a/src/yunohost/repositories/tar.py
+++ b/src/yunohost/repositories/tar.py
@@ -18,11 +18,26 @@
     along with this program; if not, see http://www.gnu.org/licenses
 
 """
+import glob
+import os
+import tarfile
+import shutil
+
+from moulinette.utils.log import getActionLogger
+from moulinette import m18n
+
+from yunohost.utils.error import YunohostError, YunohostValidationError
+from yunohost.repository import LocalBackupRepository
+from yunohost.backup import BackupManager
+from yunohost.utils.filesystem import space_used_in_directory
+from yunohost.settings import settings_get
+logger = getActionLogger("yunohost.repository")
 
 
 class TarBackupRepository(LocalBackupRepository):
     need_organized_files = False
     method_name = "tar"
+
     def list_archives_names(self):
         # Get local archives sorted according to last modification time
         # (we do a realpath() to resolve symlinks)
@@ -96,7 +111,7 @@ class TarBackupArchive:
         # Move info file
         shutil.copy(
             os.path.join(self.work_dir, "info.json"),
-            os.path.join(ARCHIVES_PATH, self.name + ".info.json"),
+            os.path.join(self.repo.location, self.name + ".info.json"),
         )
 
         # If backuped to a non-default location, keep a symlink of the archive
diff --git a/src/yunohost/repository.py b/src/yunohost/repository.py
index 8a540867b..0f40cc4d7 100644
--- a/src/yunohost/repository.py
+++ b/src/yunohost/repository.py
@@ -58,6 +58,7 @@ REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
 # TODO F2F server
 # TODO i18n pattern error
 
+
 class BackupRepository(ConfigPanel):
     """
     BackupRepository manage all repository the admin added to the instance
@@ -74,7 +75,7 @@ class BackupRepository(ConfigPanel):
         Split a repository location into protocol, user, domain and path
         """
         if "/" not in location:
-            return { "domain": location }
+            return {"domain": location}
 
         location_regex = r'^((?P<protocol>ssh://)?(?P<user>[^@ ]+)@(?P<domain>[^: ]+):((?P<port>\d+)/)?)?(?P<path>[^:]+)$'
         location_match = re.match(location_regex, location)
@@ -198,10 +199,13 @@ class BackupRepository(ConfigPanel):
 
         if self.__class__ == BackupRepository:
             if self.method == 'tar':
+                from yunohost.repositories.tar import TarBackupRepository
                 self.__class__ = TarBackupRepository
             elif self.method == 'borg':
+                from yunohost.repositories.borg import BorgBackupRepository
                 self.__class__ = BorgBackupRepository
             else:
+                from yunohost.repositories.hook import HookBackupRepository
                 self.__class__ = HookBackupRepository
 
     def _check_is_enough_free_space(self):
@@ -259,6 +263,45 @@ class BackupRepository(ConfigPanel):
 
         return archives
 
+    def prune(self, prefix=None, **kwargs):
+
+        # List archives with creation date
+        archives = {}
+        for archive_name in self.list_archive_name(prefix):
+            archive = BackupArchive(repo=self, name=archive_name)
+            created_at = archive.info()["created_at"]
+            archives[created_at] = archive
+
+        if not archives:
+            return
+
+        # Generate periods in which keep one archive
+        now = datetime.utcnow()
+        now -= timedelta(
+            minutes=now.minute,
+            seconds=now.second,
+            microseconds=now.microsecond
+        )
+        periods = set([])
+
+        for unit, qty in kwargs:
+            if not qty:
+                continue
+            period = timedelta(**{unit: 1})
+            periods += set([(now - period * i, now - period * (i - 1))
+                        for i in range(qty)])
+
+        # Delete unneeded archive
+        for created_at in sorted(archives, reverse=True):
+            created_at = datetime.utcfromtimestamp(created_at)
+            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
+
+            if keep_for:
+                periods -= keep_for
+                continue
+
+            archive.delete()
+
     # =================================================
     # Repository abstract actions
     # =================================================
@@ -277,8 +320,6 @@ class BackupRepository(ConfigPanel):
     def compute_space_used(self):
         raise NotImplementedError()
 
-    def prune(self):
-        raise NotImplementedError() # TODO prune
 
 class LocalBackupRepository(BackupRepository):
     def install(self):

From 218865a59e2361a5baddabcbb4e4abb6c7bdd4dc Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 14 Sep 2022 15:11:04 +0200
Subject: [PATCH 28/48] [fix] Pep8 and syntax

---
 src/backup.py | 157 +++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 123 insertions(+), 34 deletions(-)

diff --git a/src/backup.py b/src/backup.py
index c7b50f375..80dabbc2c 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -26,22 +26,19 @@
 import os
 import json
 import time
-import tarfile
 import shutil
 import subprocess
 import csv
 import tempfile
+import re
+import urllib
 from datetime import datetime
-from glob import glob
-from collections import OrderedDict
-from functools import reduce
 from packaging import version
 
 from moulinette import Moulinette, m18n
 from moulinette.utils import filesystem
-from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml
+from moulinette.utils.filesystem import mkdir, write_to_yaml, read_yaml, write_to_file
 from moulinette.utils.process import check_output
 
 import yunohost.domain
@@ -66,11 +63,11 @@ from yunohost.tools import (
 )
 from yunohost.regenconf import regen_conf
 from yunohost.log import OperationLogger, is_unit_operation
-from yunohost.repository import BackupRepository
+from yunohost.repository import BackupRepository, BackupArchive
+from yunohost.config import ConfigPanel
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.packages import ynh_packages_version
 from yunohost.utils.filesystem import free_space_in_directory, disk_usage, binary_to_human
-from yunohost.settings import settings_get
 
 BACKUP_PATH = "/home/yunohost.backup"
 ARCHIVES_PATH = f"{BACKUP_PATH}/archives"
@@ -113,6 +110,7 @@ class BackupRestoreTargetsManager:
             self.results[category][element] = value
         else:
             currentValue = self.results[category][element]
+
             if levels.index(currentValue) > levels.index(value):
                 return
             else:
@@ -146,6 +144,7 @@ class BackupRestoreTargetsManager:
         """
 
         # If no targets wanted, set as empty list
+
         if wanted_targets is None:
             self.targets[category] = []
 
@@ -171,6 +170,7 @@ class BackupRestoreTargetsManager:
                 error_if_wanted_target_is_unavailable(target)
 
         # For target with no result yet (like 'Skipped'), set it as unknown
+
         if self.targets[category] is not None:
             for target in self.targets[category]:
                 self.set_result(category, target, "Unknown")
@@ -192,14 +192,18 @@ class BackupRestoreTargetsManager:
         if include:
             return [
                 target
+
                 for target in self.targets[category]
+
                 if self.results[category][target] in include
             ]
 
         if exclude:
             return [
                 target
+
                 for target in self.targets[category]
+
                 if self.results[category][target] not in exclude
             ]
 
@@ -284,17 +288,18 @@ class BackupManager:
         self.targets = BackupRestoreTargetsManager()
 
         # Define backup name if needed
+
         if not name:
             name = self._define_backup_name()
         self.name = name
 
         # Define working directory if needed and initialize it
         self.work_dir = work_dir
+
         if self.work_dir is None:
             self.work_dir = os.path.join(BACKUP_PATH, "tmp", name)
         self._init_work_dir()
 
-
     #
     # Misc helpers
     #
@@ -302,6 +307,7 @@ class BackupManager:
     @property
     def info(self):
         """(Getter) Dict containing info about the archive being created"""
+
         return {
             "description": self.description,
             "created_at": self.created_at,
@@ -316,6 +322,7 @@ class BackupManager:
     def is_tmp_work_dir(self):
         """(Getter) Return true if the working directory is temporary and should
         be clean at the end of the backup"""
+
         return self.work_dir == os.path.join(BACKUP_PATH, "tmp", self.name)
 
     def __repr__(self):
@@ -328,6 +335,7 @@ class BackupManager:
             (string) A backup name created from current date 'YYMMDD-HHMMSS'
         """
         # FIXME: case where this name already exist
+
         return time.strftime("%Y%m%d-%H%M%S", time.gmtime())
 
     def _init_work_dir(self):
@@ -338,6 +346,7 @@ class BackupManager:
 
         # FIXME replace isdir by exists ? manage better the case where the path
         # exists
+
         if not os.path.isdir(self.work_dir):
             filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin")
         elif self.is_tmp_work_dir:
@@ -348,6 +357,7 @@ class BackupManager:
             )
 
             # Try to recursively unmount stuff (from a previously failed backup ?)
+
             if not _recursive_umount(self.work_dir):
                 raise YunohostValidationError("backup_output_directory_not_empty")
             else:
@@ -448,9 +458,11 @@ class BackupManager:
         # => "wordpress" dir will be put inside "sources/" and won't be renamed
 
         """
+
         if dest is None:
             dest = source
             source = os.path.join(self.work_dir, source)
+
         if dest.endswith("/"):
             dest = os.path.join(dest, os.path.basename(source))
         self.paths_to_backup.append({"source": source, "dest": dest})
@@ -538,10 +550,13 @@ class BackupManager:
         # Add unlisted files from backup tmp dir
         self._add_to_list_to_backup("backup.csv")
         self._add_to_list_to_backup("info.json")
+
         for app in self.apps_return.keys():
             self._add_to_list_to_backup(f"apps/{app}")
+
         if os.path.isdir(os.path.join(self.work_dir, "conf")):
             self._add_to_list_to_backup("conf")
+
         if os.path.isdir(os.path.join(self.work_dir, "data")):
             self._add_to_list_to_backup("data")
 
@@ -599,6 +614,7 @@ class BackupManager:
         system_targets = self.targets.list("system", exclude=["Skipped"])
 
         # If nothing to backup, return immediately
+
         if system_targets == []:
             return
 
@@ -621,14 +637,18 @@ class BackupManager:
             hook: [
                 path for path, result in infos.items() if result["state"] == "succeed"
             ]
+
             for hook, infos in ret.items()
+
             if any(result["state"] == "succeed" for result in infos.values())
         }
         ret_failed = {
             hook: [
                 path for path, result in infos.items() if result["state"] == "failed"
             ]
+
             for hook, infos in ret.items()
+
             if any(result["state"] == "failed" for result in infos.values())
         }
 
@@ -643,6 +663,7 @@ class BackupManager:
         # a restore hook available)
 
         restore_hooks_dir = os.path.join(self.work_dir, "hooks", "restore")
+
         if not os.path.exists(restore_hooks_dir):
             filesystem.mkdir(restore_hooks_dir, mode=0o700, parents=True, uid="root")
 
@@ -651,6 +672,7 @@ class BackupManager:
         for part in ret_succeed.keys():
             if part in restore_hooks:
                 part_restore_hooks = hook_info("restore", part)["hooks"]
+
                 for hook in part_restore_hooks:
                     self._add_to_list_to_backup(hook["path"], "hooks/restore/")
                 self.targets.set_result("system", part, "Success")
@@ -785,8 +807,10 @@ class BackupManager:
         # FIXME Some archive will set up dependencies, those are not in this
         # size info
         self.size = 0
+
         for system_key in self.system_return:
             self.size_details["system"][system_key] = 0
+
         for app_key in self.apps_return:
             self.size_details["apps"][app_key] = 0
 
@@ -799,10 +823,12 @@ class BackupManager:
             # Add size to apps details
             splitted_dest = row["dest"].split("/")
             category = splitted_dest[0]
+
             if category == "apps":
                 for app_key in self.apps_return:
                     if row["dest"].startswith("apps/" + app_key):
                         self.size_details["apps"][app_key] += size
+
                         break
 
             # OR Add size to the correct system element
@@ -810,6 +836,7 @@ class BackupManager:
                 for system_key in self.system_return:
                     if row["dest"].startswith(system_key.replace("_", "/")):
                         self.size_details["system"][system_key] += size
+
                         break
 
             self.size += size
@@ -897,6 +924,7 @@ class RestoreManager:
                 self.info = json.load(f)
 
             # Historically, "system" was "hooks"
+
             if "system" not in self.info.keys():
                 self.info["system"] = self.info["hooks"]
         except IOError:
@@ -916,6 +944,7 @@ class RestoreManager:
         Post install yunohost if needed
         """
         # Check if YunoHost is installed
+
         if not os.path.isfile("/etc/yunohost/installed"):
             # Retrieve the domain from the backup
             try:
@@ -945,6 +974,7 @@ class RestoreManager:
 
         if os.path.ismount(self.work_dir):
             ret = subprocess.call(["umount", self.work_dir])
+
             if ret != 0:
                 logger.warning(m18n.n("restore_cleaning_failed"))
         filesystem.rm(self.work_dir, recursive=True, force=True)
@@ -992,6 +1022,7 @@ class RestoreManager:
             # Otherwise, attempt to find it (or them?) in the archive
 
             # If we didn't find it, we ain't gonna be able to restore it
+
             if (
                 system_part not in self.info["system"]
                 or "paths" not in self.info["system"][system_part]
@@ -999,6 +1030,7 @@ class RestoreManager:
             ):
                 logger.error(m18n.n("restore_hook_unavailable", part=system_part))
                 self.targets.set_result("system", system_part, "Skipped")
+
                 continue
 
             hook_paths = self.info["system"][system_part]["paths"]
@@ -1006,6 +1038,7 @@ class RestoreManager:
 
             # Otherwise, add it from the archive to the system
             # FIXME: Refactor hook_add and use it instead
+
             for hook_path in hook_paths:
                 logger.debug(
                     "Adding restoration script '%s' to the system "
@@ -1036,6 +1069,7 @@ class RestoreManager:
         # Otherwise, if at least one app can be restored, we keep going on
         # because those which can be restored will indeed be restored
         already_installed = [app for app in to_be_restored if _is_installed(app)]
+
         if already_installed != []:
             if already_installed == to_be_restored:
                 raise YunohostValidationError(
@@ -1067,6 +1101,7 @@ class RestoreManager:
         if os.path.ismount(self.work_dir):
             logger.debug("An already mounting point '%s' already exists", self.work_dir)
             ret = subprocess.call(["umount", self.work_dir])
+
             if ret == 0:
                 subprocess.call(["rmdir", self.work_dir])
                 logger.debug(f"Unmount dir: {self.work_dir}")
@@ -1077,6 +1112,7 @@ class RestoreManager:
                 "temporary restore directory '%s' already exists", self.work_dir
             )
             ret = subprocess.call(["rm", "-Rf", self.work_dir])
+
             if ret == 0:
                 logger.debug(f"Delete dir: {self.work_dir}")
             else:
@@ -1108,8 +1144,10 @@ class RestoreManager:
 
         # If complete restore operations (or legacy archive)
         margin = CONF_MARGIN_SPACE_SIZE * 1024 * 1024
+
         if (restore_all_system and restore_all_apps) or "size_details" not in self.info:
             size = self.info["size"]
+
             if (
                 "size_details" not in self.info
                 or self.info["size_details"]["apps"] != {}
@@ -1118,11 +1156,13 @@ class RestoreManager:
         # Partial restore don't need all backup size
         else:
             size = 0
+
             if system is not None:
                 for system_element in system:
                     size += self.info["size_details"]["system"][system_element]
 
             # TODO how to know the dependencies size ?
+
             if apps is not None:
                 for app in apps:
                     size += self.info["size_details"]["apps"][app]
@@ -1130,6 +1170,7 @@ class RestoreManager:
 
         if not os.path.isfile("/etc/yunohost/installed"):
             size += POSTINSTALL_ESTIMATE_SPACE_SIZE * 1024 * 1024
+
         return (size, margin)
 
     def assert_enough_free_space(self):
@@ -1140,6 +1181,7 @@ class RestoreManager:
         free_space = free_space_in_directory(BACKUP_PATH)
 
         (needed_space, margin) = self._compute_needed_space()
+
         if free_space >= needed_space + margin:
             return True
         elif free_space > needed_space:
@@ -1200,6 +1242,7 @@ class RestoreManager:
         with open(backup_csv) as csvfile:
             reader = csv.DictReader(csvfile, fieldnames=["source", "dest"])
             newlines = []
+
             for row in reader:
                 for pattern, replace in LEGACY_PHP_VERSION_REPLACEMENTS:
                     if pattern in row["source"]:
@@ -1215,6 +1258,7 @@ class RestoreManager:
             writer = csv.DictWriter(
                 csvfile, fieldnames=["source", "dest"], quoting=csv.QUOTE_ALL
             )
+
             for row in newlines:
                 writer.writerow(row)
 
@@ -1224,6 +1268,7 @@ class RestoreManager:
         system_targets = self.targets.list("system", exclude=["Skipped"])
 
         # If nothing to restore, return immediately
+
         if system_targets == []:
             return
 
@@ -1262,12 +1307,16 @@ class RestoreManager:
 
         ret_succeed = [
             hook
+
             for hook, infos in ret.items()
+
             if any(result["state"] == "succeed" for result in infos.values())
         ]
         ret_failed = [
             hook
+
             for hook, infos in ret.items()
+
             if any(result["state"] == "failed" for result in infos.values())
         ]
 
@@ -1275,6 +1324,7 @@ class RestoreManager:
             self.targets.set_result("system", part, "Success")
 
         error_part = []
+
         for part in ret_failed:
             logger.error(m18n.n("restore_system_part_failed", part=part))
             self.targets.set_result("system", part, "Error")
@@ -1296,14 +1346,17 @@ class RestoreManager:
         )
 
         # Remove all permission for all app still in the LDAP
+
         for permission_name in user_permission_list(ignore_system_perms=True)[
             "permissions"
         ].keys():
             permission_delete(permission_name, force=True, sync_perm=False)
 
         # Restore permission for apps installed
+
         for permission_name, permission_infos in old_apps_permission.items():
             app_name, perm_name = permission_name.split(".")
+
             if _is_installed(app_name):
                 permission_create(
                     permission_name,
@@ -1312,6 +1365,7 @@ class RestoreManager:
                     additional_urls=permission_infos["additional_urls"],
                     auth_header=permission_infos["auth_header"],
                     label=permission_infos["label"]
+
                     if perm_name == "main"
                     else permission_infos["sublabel"],
                     show_tile=permission_infos["show_tile"],
@@ -1368,15 +1422,18 @@ class RestoreManager:
             for item in os.listdir(src):
                 s = os.path.join(src, item)
                 d = os.path.join(dst, item)
+
                 if os.path.isdir(s):
                     shutil.copytree(s, d, symlinks, ignore)
                 else:
                     shutil.copy2(s, d)
 
         # Check if the app is not already installed
+
         if _is_installed(app_instance_name):
             logger.error(m18n.n("restore_already_installed_app", app=app_instance_name))
             self.targets.set_result("apps", app_instance_name, "Error")
+
             return
 
         # Start register change on system
@@ -1404,9 +1461,11 @@ class RestoreManager:
 
         # Check if the app has a restore script
         app_restore_script_in_archive = os.path.join(app_scripts_in_archive, "restore")
+
         if not os.path.isfile(app_restore_script_in_archive):
             logger.warning(m18n.n("unrestore_app", app=app_instance_name))
             self.targets.set_result("apps", app_instance_name, "Warning")
+
             return
 
         try:
@@ -1427,6 +1486,7 @@ class RestoreManager:
             restore_script = os.path.join(tmp_workdir_for_app, "restore")
 
             # Restore permissions
+
             if not os.path.isfile(f"{app_settings_new_path}/permissions.yml"):
                 raise YunohostError(
                     "Didnt find a permssions.yml for the app !?", raw_msg=True
@@ -1455,6 +1515,7 @@ class RestoreManager:
                     additional_urls=permission_infos.get("additional_urls"),
                     auth_header=permission_infos.get("auth_header"),
                     label=permission_infos.get("label")
+
                     if perm_name == "main"
                     else permission_infos.get("sublabel"),
                     show_tile=permission_infos.get("show_tile", True),
@@ -1548,6 +1609,7 @@ class RestoreManager:
                 remove_operation_logger.start()
 
                 # Execute remove script
+
                 if hook_exec(remove_script, env=env_dict_remove)[0] != 0:
                     msg = m18n.n("app_not_properly_removed", app=app_instance_name)
                     logger.warning(msg)
@@ -1559,6 +1621,7 @@ class RestoreManager:
                 shutil.rmtree(app_settings_new_path, ignore_errors=True)
 
                 # Remove all permission in LDAP for this app
+
                 for permission_name in user_permission_list()["permissions"].keys():
                     if permission_name.startswith(app_instance_name + "."):
                         permission_delete(permission_name, force=True)
@@ -1577,7 +1640,7 @@ def backup_create(
     operation_logger,
     name=None,
     description=None,
-    reposistories=[],
+    repositories=[],
     system=[],
     apps=[],
     dry_run=False,
@@ -1588,7 +1651,7 @@ def backup_create(
     Keyword arguments:
         name -- Name of the backup archive
         description -- Short description of the backup
-        method -- Method of backup to use
+        repositories -- Repositories in which we want to save the backup
         output_directory -- Output directory for the backup
         system -- List of system elements to backup
         apps -- List of application names to backup
@@ -1601,10 +1664,12 @@ def backup_create(
     #
 
     # Validate there is no archive with the same name
+
     if name and name in backup_list(repositories)["archives"]:
         raise YunohostValidationError("backup_archive_name_exists")
 
     # If no --system or --apps given, backup everything
+
     if system is None and apps is None:
         system = []
         apps = []
@@ -1616,13 +1681,14 @@ def backup_create(
     operation_logger.start()
 
     # Create yunohost archives directory if it does not exists
-    _create_archive_dir() # FIXME
+    _create_archive_dir()  # FIXME
+
+    # Add backup repositories
 
-    # Add backup methods
     if not repositories:
         repositories = ["local-borg"]
 
-    repositories = [BackupRepository(repo) for repo in reposistories]
+    repositories = [BackupRepository(repo) for repo in repositories]
 
     # Prepare files to backup
     backup_manager = BackupManager(name, description,
@@ -1686,6 +1752,7 @@ def backup_restore(name, system=[], apps=[], force=False):
     #
 
     # If no --system or --apps given, restore everything
+
     if system is None and apps is None:
         system = []
         apps = []
@@ -1714,6 +1781,7 @@ def backup_restore(name, system=[], apps=[], force=False):
         "/etc/yunohost/installed"
     ):
         logger.warning(m18n.n("yunohost_already_installed"))
+
         if not force:
             try:
                 # Ask confirmation for restoring
@@ -1725,6 +1793,7 @@ def backup_restore(name, system=[], apps=[], force=False):
             else:
                 if i == "y" or i == "Y":
                     force = True
+
             if not force:
                 raise YunohostError("restore_failed")
 
@@ -1737,6 +1806,7 @@ def backup_restore(name, system=[], apps=[], force=False):
     restore_manager.restore()
 
     # Check if something has been restored
+
     if restore_manager.success:
         logger.success(m18n.n("restore_complete"))
     else:
@@ -1755,23 +1825,30 @@ def backup_list(repositories=[], with_info=False, human_readable=False):
         human_readable -- Print sizes in human readable format
 
     """
+
     return {
         name: BackupRepository(name).list(with_info)
+
         for name in repositories or BackupRepository.list(full=False)
     }
 
+
 def backup_download(name, repository):
 
-    repo = BackupRepository(repo)
+    repo = BackupRepository(repository)
     archive = BackupArchive(name, repo)
+
     return archive.download()
 
+
 def backup_mount(name, repository, path):
 
-    repo = BackupRepository(repo)
+    repo = BackupRepository(repository)
     archive = BackupArchive(name, repo)
+
     return archive.mount(path)
 
+
 def backup_info(name, repository=None, with_details=False, human_readable=False):
     """
     Get info about a local backup archive
@@ -1782,10 +1859,12 @@ def backup_info(name, repository=None, with_details=False, human_readable=False)
         human_readable -- Print sizes in human readable format
 
     """
-    repo = BackupRepository(repo)
+    repo = BackupRepository(repository)
     archive = BackupArchive(name, repo)
+
     return archive.info()
 
+
 def backup_delete(name, repository):
     """
     Delete a backup
@@ -1794,7 +1873,7 @@ def backup_delete(name, repository):
         name -- Name of the local backup archive
 
     """
-    repo = BackupRepository(repo)
+    repo = BackupRepository(repository)
     archive = BackupArchive(name, repo)
 
     # FIXME Those are really usefull ?
@@ -1807,7 +1886,6 @@ def backup_delete(name, repository):
     logger.success(m18n.n("backup_deleted"))
 
 
-
 #
 # Repository subcategory
 #
@@ -1817,7 +1895,8 @@ def backup_repository_list(full=False):
     """
     List available repositories where put archives
     """
-    return { "repositories": BackupRepository.list(full) }
+
+    return {"repositories": BackupRepository.list(full)}
 
 
 def backup_repository_info(shortname, space_used=False):
@@ -1833,11 +1912,13 @@ def backup_repository_add(operation_logger, shortname, name=None, location=None,
     """
     args = {k: v for k, v in locals().items() if v is not None}
     repository = BackupRepository(shortname, creation=True)
+
     return repository.set(
-        operation_logger=args.pop('operation_logger')
-        args=urllib.parse.urlencode(args),
+        operation_logger=args.pop('operation_logger'),
+        args=urllib.parse.urlencode(args)
     )
 
+
 @is_unit_operation()
 def backup_repository_update(operation_logger, shortname, name=None,
                              quota=None, passphrase=None,
@@ -1848,6 +1929,7 @@ def backup_repository_update(operation_logger, shortname, name=None,
 
     backup_repository_add(creation=False, **locals())
 
+
 @is_unit_operation()
 def backup_repository_remove(operation_logger, shortname, purge=False):
     """
@@ -1877,11 +1959,10 @@ class BackupTimer(ConfigPanel):
         self.values = self._get_default_values()
 
         if os.path.exists(self.save_path) and os.path.isfile(self.save_path):
-            raise NotImplementedError() # TODO
+            raise NotImplementedError()  # TODO
 
         if os.path.exists(self.service_path) and os.path.isfile(self.service_path):
-            raise NotImplementedError() # TODO
-
+            raise NotImplementedError()  # TODO
 
     def _apply(self, values):
         write_to_file(self.save_path, f"""[Unit]
@@ -1911,7 +1992,7 @@ def backup_timer_list(full=False):
     """
     List all backup timer
     """
-    return { "backup_timer": BackupTimer.list(full) }
+    return {"backup_timer": BackupTimer.list(full)}
 
 
 def backup_timer_info(shortname, space_used=False):
@@ -1921,7 +2002,7 @@ def backup_timer_info(shortname, space_used=False):
 @is_unit_operation()
 def backup_timer_add(
     operation_logger,
-    name=None,
+    shortname=None,
     description=None,
     repos=[],
     system=[],
@@ -1939,19 +2020,21 @@ def backup_timer_add(
     args = {k: v for k, v in locals().items() if v is not None}
     timer = BackupTimer(shortname, creation=True)
     return timer.set(
-        operation_logger=args.pop('operation_logger')
-        args=urllib.parse.urlencode(args),
+        operation_logger=args.pop('operation_logger'),
+        args=urllib.parse.urlencode(args)
     )
 
+
 @is_unit_operation()
 def backup_timer_update(operation_logger, shortname, name=None,
-                             quota=None, passphrase=None,
-                             alert=None, alert_delay=None):
+                        quota=None, passphrase=None,
+                        alert=None, alert_delay=None):
     """
     Update a backup timer
     """
 
-    backup_timer_add(creation=False, **locals()):
+    backup_timer_add(creation=False, **locals())
+
 
 @is_unit_operation()
 def backup_timer_remove(operation_logger, shortname, purge=False):
@@ -1961,7 +2044,6 @@ def backup_timer_remove(operation_logger, shortname, purge=False):
     BackupTimer(shortname).remove(purge)
 
 
-
 #
 # Misc helpers                                                              #
 #
@@ -1969,6 +2051,7 @@ def backup_timer_remove(operation_logger, shortname, purge=False):
 
 def _create_archive_dir():
     """Create the YunoHost archives directory if doesn't exist"""
+
     if not os.path.isdir(ARCHIVES_PATH):
         if os.path.lexists(ARCHIVES_PATH):
             raise YunohostError("backup_output_symlink_dir_broken", path=ARCHIVES_PATH)
@@ -1980,10 +2063,12 @@ def _create_archive_dir():
 
 def _call_for_each_path(self, callback, csv_path=None):
     """Call a callback for each path in csv"""
+
     if csv_path is None:
         csv_path = self.csv_path
     with open(csv_path, "r") as backup_file:
         backup_csv = csv.DictReader(backup_file, fieldnames=["source", "dest"])
+
         for row in backup_csv:
             callback(self, row["source"], row["dest"])
 
@@ -1999,17 +2084,21 @@ def _recursive_umount(directory):
 
     points_to_umount = [
         line.split(" ")[2]
+
         for line in mount_lines
+
         if len(line) >= 3 and line.split(" ")[2].startswith(os.path.realpath(directory))
     ]
 
     everything_went_fine = True
+
     for point in reversed(points_to_umount):
         ret = subprocess.call(["umount", point])
+
         if ret != 0:
             everything_went_fine = False
             logger.warning(m18n.n("backup_cleaning_failed", point))
+
             continue
 
     return everything_went_fine
-

From e0fa223d3d3f05f4920f22de245ed09d41c6ae81 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 14 Sep 2022 18:38:11 +0200
Subject: [PATCH 29/48] [fix] Pep8 and syntax

---
 src/backup.py            |  2 +-
 src/repositories/borg.py | 22 +++++++-------
 src/repositories/hook.py | 42 ++++++++++++++++++++++----
 src/repositories/tar.py  |  1 -
 src/repository.py        | 64 ++++++++++++++++++----------------------
 src/utils/config.py      | 13 ++++----
 6 files changed, 83 insertions(+), 61 deletions(-)

diff --git a/src/backup.py b/src/backup.py
index 80dabbc2c..8c4838f5e 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -31,7 +31,7 @@ import subprocess
 import csv
 import tempfile
 import re
-import urllib
+import urllib.parse
 from datetime import datetime
 from packaging import version
 
diff --git a/src/repositories/borg.py b/src/repositories/borg.py
index a9b1d4654..3fd90984e 100644
--- a/src/repositories/borg.py
+++ b/src/repositories/borg.py
@@ -22,13 +22,13 @@ import os
 import subprocess
 import json
 
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from moulinette.utils.log import getActionLogger
-from moulinette import m18n
 
 from yunohost.utils.error import YunohostError
-from yunohost.repository import LocalBackupRepository
+from yunohost.utils.network import shf_request
+from yunohost.repository import LocalBackupRepository, BackupArchive
 logger = getActionLogger("yunohost.repository")
 
 
@@ -90,14 +90,14 @@ class BorgBackupRepository(LocalBackupRepository):
                 response = shf_request(
                     domain=self.domain,
                     service=services[self.method],
-                    shf_id=values.pop('shf_id', None),
+                    shf_id=self.values.pop('shf_id', None),
                     data={
                         'origin': self.domain,
                         'public_key': self.public_key,
                         'quota': self.quota,
                         'alert': self.alert,
                         'alert_delay': self.alert_delay,
-                        #password: "XXXXXXXX",
+                        # password: "XXXXXXXX",
                     }
                 )
                 self.new_values['shf_id'] = response['id']
@@ -123,13 +123,13 @@ class BorgBackupRepository(LocalBackupRepository):
 
     def purge(self):
         if self.is_shf:
-            response = shf_request(
+            shf_request(
                 domain=self.domain,
                 service="borgbackup",
-                shf_id=values.pop('shf_id', None),
+                shf_id=self.values.pop('shf_id', None),
                 data={
                     'origin': self.domain,
-                    #password: "XXXXXXXX",
+                    # password: "XXXXXXXX",
                 }
             )
         else:
@@ -179,7 +179,7 @@ class BorgBackupRepository(LocalBackupRepository):
                 continue
             period = timedelta(**{unit: 1})
             periods += set([(now - period * i, now - period * (i - 1))
-                        for i in range(qty)])
+                           for i in range(qty)])
 
         # Delete unneeded archive
         for created_at in sorted(archives, reverse=True):
@@ -194,7 +194,7 @@ class BorgBackupRepository(LocalBackupRepository):
 
 
 class BorgBackupArchive(BackupArchive):
-        """ Backup prepared files with borg """
+    """ Backup prepared files with borg """
 
     def backup(self):
         cmd = ['borg', 'create', self.archive_path, './']
@@ -244,5 +244,3 @@ class BorgBackupArchive(BackupArchive):
         # FIXME How to be sure the place where we mount is secure ?
         cmd = ['borg', 'mount', self.archive_path, path]
         self.repo._call('mount_archive', cmd)
-
-
diff --git a/src/repositories/hook.py b/src/repositories/hook.py
index 817519162..7641b05f9 100644
--- a/src/repositories/hook.py
+++ b/src/repositories/hook.py
@@ -1,3 +1,32 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2013 Yunohost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+from moulinette import m18n
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import rm
+
+from yunohost.hook import hook_callback
+from yunohost.utils.error import YunohostError
+from yunohost.repository import BackupRepository, BackupArchive
+logger = getActionLogger("yunohost.repository")
+
 
 class HookBackupRepository(BackupRepository):
     method_name = "hook"
@@ -13,7 +42,7 @@ class HookBackupRepository(BackupRepository):
 
     def remove(self, purge=False):
         if self.__class__ == BackupRepository:
-            raise NotImplementedError() # purge
+            raise NotImplementedError()  # purge
 
         rm(self.save_path, force=True)
         logger.success(m18n.n("repository_removed", repository=self.shortname))
@@ -24,8 +53,8 @@ class HookBackupRepository(BackupRepository):
     def info(self, space_used=False):
         result = super().get(mode="export")
 
-        if self.__class__ == BackupRepository and space_used == True:
-            raise NotImplementedError() # purge
+        if self.__class__ == BackupRepository and space_used is True:
+            raise NotImplementedError()  # purge
 
         return {self.shortname: result}
 
@@ -44,7 +73,7 @@ class HookBackupArchive(BackupArchive):
         """
 
         self._call('backup', self.work_dir, self.name, self.repo.location, self.manager.size,
-                self.manager.description)
+                   self.manager.description)
 
     def restore(self):
         raise NotImplementedError()
@@ -64,7 +93,7 @@ class HookBackupArchive(BackupArchive):
         return result
 
     def info(self):
-        raise NotImplementedError() #compute_space_used
+        raise NotImplementedError()  # compute_space_used
         """ Return json string of the info.json file
 
         Exceptions:
@@ -82,7 +111,7 @@ class HookBackupArchive(BackupArchive):
         """
         super().mount()
         self._call('mount', self.work_dir, self.name, self.repo.location, self.manager.size,
-                self.manager.description)
+                   self.manager.description)
 
     def extract(self):
         raise NotImplementedError()
@@ -97,6 +126,7 @@ class HookBackupArchive(BackupArchive):
         except YunohostError:
             return False
         return True
+
     def _call(self, *args):
         """ Call a submethod of backup method hook
 
diff --git a/src/repositories/tar.py b/src/repositories/tar.py
index d49643ed7..9d0fe539d 100644
--- a/src/repositories/tar.py
+++ b/src/repositories/tar.py
@@ -235,4 +235,3 @@ class TarBackupArchive:
             if not os.path.exists(archive_file):
                 raise YunohostError('backup_archive_broken_link',
                                     path=archive_file)
-
diff --git a/src/repository.py b/src/repository.py
index 0f40cc4d7..a79bffab5 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -23,25 +23,26 @@
 
     Manage backup repositories
 """
+import json
 import os
 import re
-import time
+import shutil
 import subprocess
-import re
-import urllib.parse
+import tarfile
+import tempfile
 
 from moulinette import Moulinette, m18n
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, read_yaml, write_to_json, rm, mkdir, chmod, chown
-from moulinette.utils.network import download_text, download_json
+from moulinette.utils.filesystem import read_file, rm, mkdir
+from moulinette.utils.network import download_text
+from datetime import timedelta, datetime
 
 
-from yunohost.utils.config import ConfigPanel, Question
-from yunohost.utils.error import YunohostError
-from yunohost.utils.filesystem import space_used_in_directory, disk_usage, binary_to_human
-from yunohost.utils.network import get_ssh_public_key, shf_request, SHF_BASE_URL
-from yunohost.log import OperationLogger, is_unit_operation
+from yunohost.utils.config import ConfigPanel
+from yunohost.utils.error import YunohostError, YunohostValidationError
+from yunohost.utils.filesystem import disk_usage, binary_to_human, free_space_in_directory
+from yunohost.utils.network import get_ssh_public_key, SHF_BASE_URL
 
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_DIR = '/etc/yunohost/repositories'
@@ -104,12 +105,11 @@ class BackupRepository(ConfigPanel):
         for repo in repositories:
             try:
                 repositories[repo] = BackupRepository(repo).info(space_used)
-            except Exception as e:
+            except Exception:
                 logger.error(f"Unable to open repository {repo}")
 
         return repositories
 
-
     # =================================================
     # Config Panel Hooks
     # =================================================
@@ -117,19 +117,18 @@ class BackupRepository(ConfigPanel):
     def post_ask__domain(self, question):
         """ Detect if the domain support Self-Hosting Federation protocol
         """
-        #import requests
+        # import requests
         # FIXME What if remote server is self-signed ?
         # FIXME What if remote server is unreachable temporarily ?
         url = SHF_BASE_URL.format(domain=question.value) + "/"
         try:
-            #r = requests.get(url, timeout=10)
+            # r = requests.get(url, timeout=10)
             download_text(url, timeout=10)
-        except MoulinetteError as e:
+        except MoulinetteError:
             logger.debug("SHF not running")
-            return { 'is_shf': False }
+            return {'is_shf': False}
         logger.debug("SHF running")
-        return { 'is_shf': True }
-
+        return {'is_shf': True}
 
     # =================================================
     # Config Panel Override
@@ -156,7 +155,7 @@ class BackupRepository(ConfigPanel):
 
     def _parse_pre_answered(self, *args):
         super()._parse_pre_answered(*args)
-        if 'location'  in self.args:
+        if 'location' in self.args:
             self.args.update(BackupRepository.split_location(self.args['location']))
         if 'domain' in self.args:
             self.args['is_remote'] = bool(self.args['domain'])
@@ -179,7 +178,6 @@ class BackupRepository(ConfigPanel):
             self.new_values.pop(prop, None)
         super()._apply()
 
-
     # =================================================
     # BackupMethod encapsulation
     # =================================================
@@ -237,7 +235,7 @@ class BackupRepository(ConfigPanel):
     def info(self, space_used=False):
         result = super().get(mode="export")
 
-        if self.__class__ == BackupRepository and space_used == True:
+        if self.__class__ == BackupRepository and space_used is True:
             result["space_used"] = self.compute_space_used()
 
         return {self.shortname: result}
@@ -245,7 +243,7 @@ class BackupRepository(ConfigPanel):
     def list(self, with_info):
         archives = self.list_archive_name()
         if with_info:
-            d = OrderedDict()
+            d = {}
             for archive in archives:
                 try:
                     d[archive] = BackupArchive(repo=self, name=archive).info()
@@ -289,7 +287,7 @@ class BackupRepository(ConfigPanel):
                 continue
             period = timedelta(**{unit: 1})
             periods += set([(now - period * i, now - period * (i - 1))
-                        for i in range(qty)])
+                           for i in range(qty)])
 
         # Delete unneeded archive
         for created_at in sorted(archives, reverse=True):
@@ -412,7 +410,7 @@ class BackupArchive:
             raise YunohostError(
                 "backup_archive_cant_retrieve_info_json", archive=self.archive_path
             )
-        extract_paths = []
+
         if f"{leading_dot}backup.csv" in files_in_archive:
             yield f"{leading_dot}backup.csv"
         else:
@@ -447,7 +445,7 @@ class BackupArchive:
         if not os.path.lexists(archive_file):
             archive_file += ".gz"
             if not os.path.lexists(archive_file):
-                raise YunohostValidationError("backup_archive_name_unknown", name=name)
+                raise YunohostValidationError("backup_archive_name_unknown", name=self.name)
 
         # If symlink, retrieve the real path
         if os.path.islink(archive_file):
@@ -491,7 +489,7 @@ class BackupArchive:
             raise YunohostError('backup_info_json_not_implemented')
         try:
             info = json.load(info_json)
-        except:
+        except Exception:
             logger.debug("unable to load info json", exc_info=1)
             raise YunohostError('backup_invalid_archive')
 
@@ -558,7 +556,8 @@ class BackupArchive:
                 raise YunohostError("backup_cleaning_failed")
 
         if self.manager.is_tmp_work_dir:
-            filesystem.rm(self.work_dir, True, True)
+            rm(self.work_dir, True, True)
+
     def _organize_files(self):
         """
         Mount all csv src in their related path
@@ -587,11 +586,11 @@ class BackupArchive:
 
             # Be sure the parent dir of destination exists
             if not os.path.isdir(dest_dir):
-                filesystem.mkdir(dest_dir, parents=True)
+                mkdir(dest_dir, parents=True)
 
             # For directory, attempt to mount bind
             if os.path.isdir(src):
-                filesystem.mkdir(dest, parents=True, force=True)
+                mkdir(dest, parents=True, force=True)
 
                 try:
                     subprocess.check_call(["mount", "--rbind", src, dest])
@@ -688,7 +687,6 @@ class BackupArchive:
         if self.__class__ == BackupArchive:
             raise NotImplementedError()
 
-
     def download(self):
         if self.__class__ == BackupArchive:
             raise NotImplementedError()
@@ -712,9 +710,3 @@ class BackupArchive:
     def mount(self):
         if self.__class__ == BackupArchive:
             raise NotImplementedError()
-
-
-
-
-
-
diff --git a/src/utils/config.py b/src/utils/config.py
index c92de7f36..6f91c5a39 100644
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -49,10 +49,13 @@ from yunohost.log import OperationLogger
 logger = getActionLogger("yunohost.config")
 CONFIG_PANEL_VERSION_SUPPORTED = 1.0
 
-# Those js-like evaluate functions are used to eval safely visible attributes
-# The goal is to evaluate in the same way than js simple-evaluate
-# https://github.com/shepherdwind/simple-evaluate
+
 def evaluate_simple_ast(node, context=None):
+    """
+    Those js-like evaluate functions are used to eval safely visible attributes
+    The goal is to evaluate in the same way than js simple-evaluate
+    https://github.com/shepherdwind/simple-evaluate
+    """
     if context is None:
         context = {}
 
@@ -1130,9 +1133,9 @@ class DomainQuestion(Question):
     @staticmethod
     def normalize(value, option={}):
         if value.startswith("https://"):
-            value = value[len("https://") :]
+            value = value[len("https://"):]
         elif value.startswith("http://"):
-            value = value[len("http://") :]
+            value = value[len("http://"):]
 
         # Remove trailing slashes
         value = value.rstrip("/").lower()

From 709fa5d4f6f686883160f1eced1222ab23218772 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 30 Sep 2022 13:20:16 +0200
Subject: [PATCH 30/48] [fix] Use correct variable in backup cron

---
 src/backup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/backup.py b/src/backup.py
index 8c4838f5e..f8fa6474a 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -902,7 +902,7 @@ class RestoreManager:
         self.targets = BackupRestoreTargetsManager()
 
     #
-    # Misc helpers                                                          #
+    # Misc helpers
     #
 
     @property
@@ -1982,7 +1982,7 @@ After=network.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/yunohost backup create -n '{name}' -r '{repo}' --system --apps ; /usr/bin/yunohost backup prune -n '{name}'
+ExecStart=/usr/bin/yunohost backup create -n '{self.entity}' -r '{repo}' --system --apps ; /usr/bin/yunohost backup prune -n '{self.entity}'
 User=root
 Group=root
 """)

From e8c0be1e56c82d8c101cd5fe834b1761b254e5aa Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 10 Oct 2022 02:13:15 +0200
Subject: [PATCH 31/48] [fix] Syntax and import error

---
 share/actionsmap.yml                          |  3 +
 ...ory.toml => config_backup_repository.toml} |  0
 share/config_backup_timer.toml                | 89 +++++++++++++++++++
 src/backup.py                                 | 82 +++++++----------
 src/repository.py                             | 44 +++++----
 5 files changed, 143 insertions(+), 75 deletions(-)
 rename share/{config_repository.toml => config_backup_repository.toml} (100%)
 create mode 100644 share/config_backup_timer.toml

diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index 908c32f22..d4fadf1d5 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1021,6 +1021,9 @@ backup:
             arguments:
                 name:
                     help: Name of the local backup archive
+                -r:
+                    full: --repository
+                    help: The archive repository (local borg repo use by default)
                 --system:
                     help: List of system parts to restore (or all if none is given)
                     nargs: "*"
diff --git a/share/config_repository.toml b/share/config_backup_repository.toml
similarity index 100%
rename from share/config_repository.toml
rename to share/config_backup_repository.toml
diff --git a/share/config_backup_timer.toml b/share/config_backup_timer.toml
new file mode 100644
index 000000000..cc0c5290f
--- /dev/null
+++ b/share/config_backup_timer.toml
@@ -0,0 +1,89 @@
+
+version = "1.0"
+i18n = "repository_config"
+[main]
+name.en = ""
+    [main.main]
+    name.en = ""
+    optional = false
+        # if method == "tar": question["value"] = False
+        [main.main.description]
+        type = "string"
+        default = ""
+
+        [main.main.is_remote]
+        type = "boolean"
+        yes = true
+        no = false
+        visible = "creation"
+        default = "no"
+
+        [main.main.domain]
+        type = "string"
+        visible = "creation && is_remote"
+        pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
+        pattern.error = 'domain_error' # TODO "Please provide a valid domain"
+        default = ""
+        # FIXME: can't be a domain of this instances ?
+
+        [main.main.is_shf]
+        help = ""
+        type = "boolean"
+        yes = true
+        no = false
+        visible = "creation && is_remote"
+        default = false
+
+        [main.main.public_key]
+        type = "alert"
+        style = "info"
+        visible = "creation && is_remote && ! is_shf"
+
+        [main.main.alert]
+        help = ''
+        type = "tags"
+        visible = "is_remote && is_shf"
+        pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
+        pattern.error = "alert_error"
+        default = []
+        # "value": alert,
+
+        [main.main.alert_delay]
+        help = ''
+        type = "number"
+        visible = "is_remote && is_shf"
+        min = 1
+        default = 7
+
+        [main.main.quota]
+        type = "string"
+        visible = "is_remote && is_shf"
+        pattern.regexp = '^\d+[MGT]$'
+        pattern.error = '' # TODO ""
+        default = ""
+
+        [main.main.port]
+        type = "number"
+        visible = "is_remote && !is_shf"
+        min = 1
+        max = 65535
+        default = 22
+
+        [main.main.user]
+        type = "string"
+        visible = "is_remote && !is_shf"
+        default = ""
+
+        [main.main.method]
+        type = "select"
+        # "value": method,
+        choices.borg = "BorgBackup (recommended)"
+        choices.tar = "Legacy tar archive mechanism"
+        default = "borg"
+        visible = "!is_remote"
+        
+        [main.main.path]
+        type = "path"
+        visible = "!is_remote or (is_remote and !is_shf)"
+        default = "/home/yunohost.backup/archives"
+
diff --git a/src/backup.py b/src/backup.py
index f8fa6474a..42fdc2416 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -32,13 +32,12 @@ import csv
 import tempfile
 import re
 import urllib.parse
-from datetime import datetime
 from packaging import version
 
 from moulinette import Moulinette, m18n
 from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import mkdir, write_to_yaml, read_yaml, write_to_file
+from moulinette.utils.filesystem import mkdir, write_to_yaml, read_yaml, write_to_file, rm
 from moulinette.utils.process import check_output
 
 import yunohost.domain
@@ -64,7 +63,7 @@ from yunohost.tools import (
 from yunohost.regenconf import regen_conf
 from yunohost.log import OperationLogger, is_unit_operation
 from yunohost.repository import BackupRepository, BackupArchive
-from yunohost.config import ConfigPanel
+from yunohost.utils.config import ConfigPanel
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.packages import ynh_packages_version
 from yunohost.utils.filesystem import free_space_in_directory, disk_usage, binary_to_human
@@ -74,7 +73,6 @@ ARCHIVES_PATH = f"{BACKUP_PATH}/archives"
 APP_MARGIN_SPACE_SIZE = 100  # In MB
 CONF_MARGIN_SPACE_SIZE = 10  # IN MB
 POSTINSTALL_ESTIMATE_SPACE_SIZE = 5  # In MB
-MB_ALLOWED_TO_ORGANIZE = 10
 logger = getActionLogger("yunohost.backup")
 
 
@@ -367,6 +365,14 @@ class BackupManager:
                 filesystem.rm(self.work_dir, recursive=True, force=True)
                 filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin")
 
+    def clean_work_dir(self, umount=True):
+
+        if umount and not _recursive_umount(self.work_dir):
+            raise YunohostError("backup_cleaning_failed")
+
+        if self.is_tmp_work_dir:
+            rm(self.work_dir, True, True)
+
     #
     # Backup target management                                              #
     #
@@ -875,18 +881,19 @@ class RestoreManager:
         return restore_manager.result
     """
 
-    def __init__(self, name, method="tar"):
+    def __init__(self, archive):
         """
         RestoreManager constructor
 
         Args:
-        name -- (string) Archive name
-        method -- (string) Method name to use to mount the archive
+        archive -- (BackupArchive) The archive to restore
         """
         # Retrieve and open the archive
         # FIXME this way to get the info is not compatible with copy or custom
+        self.archive = archive
+
         # backup methods
-        self.info = backup_info(name, with_details=True)
+        self.info = archive.info()  # FIXME with_details=True
 
         from_version = self.info.get("from_yunohost_version", "")
         # Remove any '~foobar' in the version ... c.f ~alpha, ~beta version during
@@ -896,9 +903,6 @@ class RestoreManager:
         if not from_version or version.parse(from_version) < version.parse("4.2.0"):
             raise YunohostValidationError("restore_backup_too_old")
 
-        self.archive_path = self.info["path"]
-        self.name = name
-        self.method = BackupMethod.create(method, self)
         self.targets = BackupRestoreTargetsManager()
 
     #
@@ -913,32 +917,6 @@ class RestoreManager:
 
         return len(successful_apps) != 0 or len(successful_system) != 0
 
-    def _read_info_files(self):
-        """
-        Read the info file from inside an archive
-        """
-        # Retrieve backup info
-        info_file = os.path.join(self.work_dir, "info.json")
-        try:
-            with open(info_file, "r") as f:
-                self.info = json.load(f)
-
-            # Historically, "system" was "hooks"
-
-            if "system" not in self.info.keys():
-                self.info["system"] = self.info["hooks"]
-        except IOError:
-            logger.debug("unable to load '%s'", info_file, exc_info=1)
-            raise YunohostError(
-                "backup_archive_cant_retrieve_info_json", archive=self.archive_path
-            )
-        else:
-            logger.debug(
-                "restoring from backup '%s' created on %s",
-                self.name,
-                datetime.utcfromtimestamp(self.info["created_at"]),
-            )
-
     def _postinstall_if_needed(self):
         """
         Post install yunohost if needed
@@ -1041,10 +1019,8 @@ class RestoreManager:
 
             for hook_path in hook_paths:
                 logger.debug(
-                    "Adding restoration script '%s' to the system "
-                    "from the backup archive '%s'",
-                    hook_path,
-                    self.archive_path,
+                    f"Adding restoration script '{hook_path}' to the system "
+                    f"from the backup archive '{self.archive.archive_path}'"
                 )
                 self.method.copy(hook_path, custom_restore_hook_folder)
 
@@ -1096,7 +1072,7 @@ class RestoreManager:
         this archive
         """
 
-        self.work_dir = os.path.join(BACKUP_PATH, "tmp", self.name)
+        self.work_dir = os.path.join(BACKUP_PATH, "tmp", self.archive.name)
 
         if os.path.ismount(self.work_dir):
             logger.debug("An already mounting point '%s' already exists", self.work_dir)
@@ -1120,9 +1096,7 @@ class RestoreManager:
 
         filesystem.mkdir(self.work_dir, parents=True)
 
-        self.method.extract()
-
-        self._read_info_files()
+        self.archive.extract()
 
     #
     # Space computation / checks                                            #
@@ -1736,7 +1710,7 @@ def backup_create(
     }
 
 
-def backup_restore(name, system=[], apps=[], force=False):
+def backup_restore(name, repository, system=[], apps=[], force=False):
     """
     Restore from a local backup archive
 
@@ -1757,6 +1731,9 @@ def backup_restore(name, system=[], apps=[], force=False):
         system = []
         apps = []
 
+    if not repository:
+        repository = "local-borg"
+
     #
     # Initialize                                                            #
     #
@@ -1766,7 +1743,10 @@ def backup_restore(name, system=[], apps=[], force=False):
     elif name.endswith(".tar"):
         name = name[: -len(".tar")]
 
-    restore_manager = RestoreManager(name)
+    repo = BackupRepository(repository)
+    archive = BackupArchive(name, repo)
+
+    restore_manager = RestoreManager(archive)
 
     restore_manager.set_system_targets(system)
     restore_manager.set_apps_targets(apps)
@@ -1827,7 +1807,7 @@ def backup_list(repositories=[], with_info=False, human_readable=False):
     """
 
     return {
-        name: BackupRepository(name).list(with_info)
+        name: BackupRepository(name).list_archives(with_info)
 
         for name in repositories or BackupRepository.list(full=False)
     }
@@ -1862,7 +1842,7 @@ def backup_info(name, repository=None, with_details=False, human_readable=False)
     repo = BackupRepository(repository)
     archive = BackupArchive(name, repo)
 
-    return archive.info()
+    return archive.info(with_details=with_details, human_readable=human_readable)
 
 
 def backup_delete(name, repository):
@@ -1969,7 +1949,7 @@ class BackupTimer(ConfigPanel):
 Description=Run backup {self.entity} regularly
 
 [Timer]
-OnCalendar={values['schedule']}
+OnCalendar={self.values['schedule']}
 
 [Install]
 WantedBy=timers.target
@@ -1982,7 +1962,7 @@ After=network.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/yunohost backup create -n '{self.entity}' -r '{repo}' --system --apps ; /usr/bin/yunohost backup prune -n '{self.entity}'
+ExecStart=/usr/bin/yunohost backup create -n '{self.entity}' -r '{self.repositories}' --system --apps ; /usr/bin/yunohost backup prune -n '{self.entity}'
 User=root
 Group=root
 """)
diff --git a/src/repository.py b/src/repository.py
index a79bffab5..5af841591 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -30,6 +30,7 @@ import shutil
 import subprocess
 import tarfile
 import tempfile
+from functools import reduce
 
 from moulinette import Moulinette, m18n
 from moulinette.core import MoulinetteError
@@ -38,7 +39,7 @@ from moulinette.utils.filesystem import read_file, rm, mkdir
 from moulinette.utils.network import download_text
 from datetime import timedelta, datetime
 
-
+import yunohost.repositories
 from yunohost.utils.config import ConfigPanel
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.filesystem import disk_usage, binary_to_human, free_space_in_directory
@@ -48,6 +49,7 @@ logger = getActionLogger('yunohost.repository')
 REPOSITORIES_DIR = '/etc/yunohost/repositories'
 CACHE_INFO_DIR = "/var/cache/yunohost/{repository}"
 REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
+MB_ALLOWED_TO_ORGANIZE = 10
 # TODO split ConfigPanel.get to extract "Format result" part and be able to override it
 # TODO Migration
 # TODO Remove BackupRepository.get_or_create()
@@ -240,7 +242,7 @@ class BackupRepository(ConfigPanel):
 
         return {self.shortname: result}
 
-    def list(self, with_info):
+    def list_archives(self, with_info):
         archives = self.list_archive_name()
         if with_info:
             d = {}
@@ -343,14 +345,14 @@ class BackupArchive:
 
         # Cast
         if self.repo.method_name == 'tar':
-            self.__class__ = TarBackupArchive
+            self.__class__ = yunohost.repositories.tar.TarBackupArchive
         elif self.repo.method_name == 'borg':
-            self.__class__ = BorgBackupArchive
+            self.__class__ = yunohost.repositories.borg.BorgBackupArchive
         else:
-            self.__class__ = HookBackupArchive
+            self.__class__ = yunohost.repositories.hook.HookBackupArchive
 
         # Assert archive exists
-        if not isinstance(self.manager, BackupManager) and self.name not in self.repo.list():
+        if self.manager.__class__.__name__ != "BackupManager" and self.name not in self.repo.list_archives():
             raise YunohostValidationError("backup_archive_name_unknown", name=name)
 
     @property
@@ -439,17 +441,17 @@ class BackupArchive:
             yield f"{leading_dot}apps/{app}"
 
     def _get_info_string(self):
-        archive_file = "%s/%s.tar" % (self.repo.path, self.name)
+        self.archive_file = "%s/%s.tar" % (self.repo.path, self.name)
 
         # Check file exist (even if it's a broken symlink)
-        if not os.path.lexists(archive_file):
-            archive_file += ".gz"
-            if not os.path.lexists(archive_file):
+        if not os.path.lexists(self.archive_file):
+            self.archive_file += ".gz"
+            if not os.path.lexists(self.archive_file):
                 raise YunohostValidationError("backup_archive_name_unknown", name=self.name)
 
         # If symlink, retrieve the real path
-        if os.path.islink(archive_file):
-            archive_file = os.path.realpath(archive_file)
+        if os.path.islink(self.archive_file):
+            archive_file = os.path.realpath(self.archive_file)
 
             # Raise exception if link is broken (e.g. on unmounted external storage)
             if not os.path.exists(archive_file):
@@ -470,7 +472,7 @@ class BackupArchive:
                     self.extract("./info.json")
                 else:
                     raise YunohostError(
-                        "backup_archive_cant_retrieve_info_json", archive=archive_file
+                        "backup_archive_cant_retrieve_info_json", archive=self.archive_file
                     )
                 shutil.move(os.path.join(info_dir, "info.json"), info_file)
             finally:
@@ -482,7 +484,7 @@ class BackupArchive:
             logger.debug("unable to load '%s'", info_file, exc_info=1)
             raise YunohostError('backup_invalid_archive')
 
-    def info(self):
+    def info(self, with_details, human_readable):
 
         info_json = self._get_info_string()
         if not self._info_json:
@@ -498,14 +500,14 @@ class BackupArchive:
         size = info.get("size", 0)
         if not size:
             tar = tarfile.open(
-                archive_file, "r:gz" if archive_file.endswith(".gz") else "r"
+                self.archive_file, "r:gz" if self.archive_file.endswith(".gz") else "r"
             )
             size = reduce(
                 lambda x, y: getattr(x, "size", x) + getattr(y, "size", y), tar.getmembers()
             )
             tar.close()
         result = {
-            "path": repo.archive_path,
+            "path": self.repo.archive_path,
             "created_at": datetime.utcfromtimestamp(info["created_at"]),
             "description": info["description"],
             "size": size,
@@ -546,17 +548,11 @@ class BackupArchive:
 
         return info
 
-# TODO move this in BackupManager ?????
     def clean(self):
         """
         Umount sub directories of working dirextories and delete it if temporary
         """
-        if self.need_organized_files():
-            if not _recursive_umount(self.work_dir):
-                raise YunohostError("backup_cleaning_failed")
-
-        if self.manager.is_tmp_work_dir:
-            rm(self.work_dir, True, True)
+        self.manager.clean_work_dir(self.need_organized_files())
 
     def _organize_files(self):
         """
@@ -573,7 +569,7 @@ class BackupArchive:
         for path in self.manager.paths_to_backup:
             src = path["source"]
 
-            if self.manager is RestoreManager:
+            if self.manager.__class__.__name__ != "RestoreManager":
                 # TODO Support to run this before a restore (and not only before
                 # backup). To do that RestoreManager.unorganized_work_dir should
                 # be implemented

From b6f2ec7a57882ea04a2350d93a0554d9b2a10d25 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 10 Oct 2022 15:27:55 +0200
Subject: [PATCH 32/48] [enh] Add borgbackup dependencies

---
 debian/control | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/control b/debian/control
index 0760e2cde..82f269474 100644
--- a/debian/control
+++ b/debian/control
@@ -29,6 +29,7 @@ Depends: ${python3:Depends}, ${misc:Depends}
  , acl
  , git, curl, wget, cron, unzip, jq, bc, at
  , lsb-release, haveged, fake-hwclock, equivs, lsof, whois
+ , borgbackup
 Recommends: yunohost-admin
  , ntp, inetutils-ping | iputils-ping
  , bash-completion, rsyslog

From 533ed0ce4741a3f3e3dacfe5949854bbe0cda4da Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 10 Oct 2022 15:28:45 +0200
Subject: [PATCH 33/48] [fix] Quota and encryption on borg repo

---
 src/repositories/borg.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/repositories/borg.py b/src/repositories/borg.py
index 3fd90984e..e5869113f 100644
--- a/src/repositories/borg.py
+++ b/src/repositories/borg.py
@@ -112,10 +112,12 @@ class BorgBackupRepository(LocalBackupRepository):
             super().install()
 
         # Initialize borg repo
-        cmd = ["borg", "init", "--encryption", "repokey", self.location]
+        encryption_mode = "repokey" if "passphrase" in self.future_values and self.future_values["passphrase"] else "none"
+        cmd = ["borg", "init", "--encryption", encryption_mode, self.location]
 
-        if "quota" in self.future_values:
+        if "quota" in self.future_values and self.future_values["quota"]:
             cmd += ['--storage-quota', self.quota]
+
         self._call('init', cmd)
 
     def update(self):

From 0d9e94df7abe4aa1a9c4a214f7ba493923efb194 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 10 Oct 2022 16:08:52 +0200
Subject: [PATCH 34/48] [fix] Missing translation

---
 locales/en.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/locales/en.json b/locales/en.json
index d8d64c8ff..5b8fb91b2 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -90,6 +90,7 @@
     "backup_archive_system_part_not_available": "System part '{part}' unavailable in this backup",
     "backup_archive_writing_error": "Could not add the files '{source}' (named in the archive '{dest}') to be backed up into the compressed archive '{archive}'",
     "backup_ask_for_copying_if_needed": "Do you want to perform the backup using {size}MB temporarily? (This way is used since some files could not be prepared using a more efficient method.)",
+    "backup_borg_init_error": "Unable initialize the borg repository",
     "backup_cant_mount_uncompress_archive": "Could not mount the uncompressed archive as write protected",
     "backup_cleaning_failed": "Could not clean up the temporary backup folder",
     "backup_copying_to_organize_the_archive": "Copying {size}MB to organize the archive",
@@ -709,4 +710,4 @@
     "yunohost_installing": "Installing YunoHost...",
     "yunohost_not_installed": "YunoHost is not correctly installed. Please run 'yunohost tools postinstall'",
     "yunohost_postinstall_end_tip": "The post-install completed! To finalize your setup, please consider:\n    - adding a first user through the 'Users' section of the webadmin (or 'yunohost user create <username>' in command-line);\n    - diagnose potential issues through the 'Diagnosis' section of the webadmin (or 'yunohost diagnosis run' in command-line);\n    - reading the 'Finalizing your setup' and 'Getting to know YunoHost' parts in the admin documentation: https://yunohost.org/admindoc."
-}
\ No newline at end of file
+}

From f292ed87fd7eeffe2dfc95186d094f3dea8083f2 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 10 Oct 2022 16:09:22 +0200
Subject: [PATCH 35/48] [fix] Config panel list method is broken

---
 src/utils/config.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/utils/config.py b/src/utils/config.py
index 6f91c5a39..e767816a8 100644
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -209,7 +209,7 @@ class ConfigPanel:
         try:
             entities = [
                 re.match(
-                    "^" + cls.save_path_tpl.format(entity="(?p<entity>)") + "$", f
+                    "^" + cls.save_path_tpl.format(entity="(?P<entity>[^/]*)") + "$", f
                 ).group("entity")
                 for f in glob.glob(cls.save_path_tpl.format(entity="*"))
                 if os.path.isfile(f)
@@ -650,7 +650,7 @@ class ConfigPanel:
         logger.info("Saving the new configuration...")
         dir_path = os.path.dirname(os.path.realpath(self.save_path))
         if not os.path.exists(dir_path):
-            mkdir(dir_path, mode=0o700)
+            mkdir(dir_path, mode=0o700, parents=True)
 
         values_to_save = self.future_values
         if self.save_mode == "diff":

From 6eca4bff69fb0a7f42aa9383960e4a8404de6769 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 10 Oct 2022 16:10:39 +0200
Subject: [PATCH 36/48] [fix] Bad repository config path

---
 src/repository.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/repository.py b/src/repository.py
index 5af841591..7b1820853 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -67,7 +67,7 @@ class BackupRepository(ConfigPanel):
     BackupRepository manage all repository the admin added to the instance
     """
     entity_type = "backup_repository"
-    save_path_tpl = "/etc/yunohost/backup/repositories/{entity}.yml"
+    save_path_tpl = REPOSITORIES_DIR + "/{entity}.yml"
     save_mode = "full"
     need_organized_files = True
     method_name = ""

From 04f85eb860ee20eca3553210ce8bb93ec447f918 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 11 Oct 2022 02:24:05 +0200
Subject: [PATCH 37/48] [fix] Repository list, add, remove

---
 locales/en.json          |  5 ++++-
 share/actionsmap.yml     |  9 ++++-----
 src/backup.py            |  4 ++--
 src/repositories/borg.py | 41 ++++++++++++++++++++++++++--------------
 src/repository.py        | 22 +++++++++++----------
 5 files changed, 49 insertions(+), 32 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index 5b8fb91b2..a3aba5dd3 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -90,7 +90,8 @@
     "backup_archive_system_part_not_available": "System part '{part}' unavailable in this backup",
     "backup_archive_writing_error": "Could not add the files '{source}' (named in the archive '{dest}') to be backed up into the compressed archive '{archive}'",
     "backup_ask_for_copying_if_needed": "Do you want to perform the backup using {size}MB temporarily? (This way is used since some files could not be prepared using a more efficient method.)",
-    "backup_borg_init_error": "Unable initialize the borg repository",
+    "backup_borg_init_error": "Unable initialize the borg repository: {error}",
+    "backup_borg_already_initialized": "The borg repository '{repository}' already exists, it has been properly added to repositories managed by YunoHost cli.",
     "backup_cant_mount_uncompress_archive": "Could not mount the uncompressed archive as write protected",
     "backup_cleaning_failed": "Could not clean up the temporary backup folder",
     "backup_copying_to_organize_the_archive": "Copying {size}MB to organize the archive",
@@ -116,6 +117,8 @@
     "backup_output_directory_required": "You must provide an output directory for the backup",
     "backup_output_symlink_dir_broken": "Your archive directory '{path}' is a broken symlink. Maybe you forgot to re/mount or plug in the storage medium it points to.",
     "backup_permission": "Backup permission for {app}",
+    "backup_repository_exists": "Backup repository '{backup_repository}' already exists",
+    "backup_repository_unknown": "Backup repository '{backup_repository}' unknown",
     "backup_running_hooks": "Running backup hooks...",
     "backup_system_part_failed": "Could not backup the '{part}' system part",
     "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive",
diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index d4fadf1d5..0b3024c43 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1099,6 +1099,9 @@ backup:
                         --full:
                             help: Show more details
                             action: store_true
+                        --space-used:
+                            help: Display size used
+                            action: store_true
                 
                 ### backup_repository_info()
                 info:
@@ -1111,13 +1114,9 @@ backup:
                                 pattern: &pattern_backup_repository_shortname
                                     - !!str ^[a-zA-Z0-9-_\.]+$
                                     - "pattern_backup_repository_shortname"
-                        -H:
-                            full: --human-readable
-                            help: Print sizes in human readable format
-                            action: store_true
                         --space-used:
                             help: Display size used
-                            action: store_false
+                            action: store_true
                 
                 ### backup_repository_add()
                 add:
diff --git a/src/backup.py b/src/backup.py
index 42fdc2416..d0bbc89d4 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -1871,12 +1871,12 @@ def backup_delete(name, repository):
 #
 
 
-def backup_repository_list(full=False):
+def backup_repository_list(space_used=False, full=False):
     """
     List available repositories where put archives
     """
 
-    return {"repositories": BackupRepository.list(full)}
+    return {"repositories": BackupRepository.list(space_used, full)}
 
 
 def backup_repository_info(shortname, space_used=False):
diff --git a/src/repositories/borg.py b/src/repositories/borg.py
index e5869113f..894d0b7ca 100644
--- a/src/repositories/borg.py
+++ b/src/repositories/borg.py
@@ -24,6 +24,7 @@ import json
 
 from datetime import datetime, timedelta
 
+from moulinette import m18n
 from moulinette.utils.log import getActionLogger
 
 from yunohost.utils.error import YunohostError
@@ -37,7 +38,7 @@ class BorgBackupRepository(LocalBackupRepository):
     method_name = "borg"
 
     # TODO logs
-    def _run_borg_command(self, cmd, stdout=None):
+    def _run_borg_command(self, cmd, stdout=None, stderr=None):
         """ Call a submethod of borg with the good context
         """
         env = dict(os.environ)
@@ -59,15 +60,16 @@ class BorgBackupRepository(LocalBackupRepository):
         # Authorize to move the repository (borgbase do this)
         env["BORG_RELOCATED_REPO_ACCESS_IS_OK"] = "yes"
 
-        return subprocess.Popen(cmd, env=env, stdout=stdout)
+        return subprocess.Popen(cmd, env=env,
+                                stdout=stdout, stderr=stderr)
 
     def _call(self, action, cmd, json_output=False):
-        borg = self._run_borg_command(cmd)
-        return_code = borg.wait()
-        if return_code:
-            raise YunohostError(f"backup_borg_{action}_error")
+        borg = self._run_borg_command(cmd, stdout=subprocess.PIPE,
+                                      stderr=subprocess.PIPE)
+        out, err = borg.communicate()
+        if borg.returncode:
+            raise YunohostError(f"backup_borg_{action}_error", error=err)
 
-        out, _ = borg.communicate()
         if json_output:
             try:
                 return json.loads(out)
@@ -117,8 +119,20 @@ class BorgBackupRepository(LocalBackupRepository):
 
         if "quota" in self.future_values and self.future_values["quota"]:
             cmd += ['--storage-quota', self.quota]
+        try:
+            self._call('init', cmd)
+        except YunohostError as e:
+            if e.key != "backup_borg_init_error":
+                raise
+            else:
+                # Check if it's possible to read the borg repo with current settings
+                try:
+                    cmd = ["borg", "info", self.location]
+                    self._call('info', cmd)
+                except YunohostError:
+                    raise e
 
-        self._call('init', cmd)
+                logger.info(m18n.n("backup_borg_already_initialized", repository=self.location))
 
     def update(self):
         raise NotImplementedError()
@@ -148,12 +162,11 @@ class BorgBackupRepository(LocalBackupRepository):
         return [archive["name"] for archive in response['archives']]
 
     def compute_space_used(self):
-        if not self.is_remote:
-            return super().purge()
-        else:
-            cmd = ["borg", "info", "--json", self.location]
-            response = self._call('info', cmd)
-            return response["cache"]["stats"]["unique_size"]
+        """ Return the size of this repo on the disk"""
+        # FIXME this size could be unrelevant, comparison between du and borg sizes doesn't match !
+        cmd = ["borg", "info", "--json", self.location]
+        response = self._call('info', cmd, json_output=True)
+        return response["cache"]["stats"]["unique_size"]
 
     def prune(self, prefix=None, **kwargs):
 
diff --git a/src/repository.py b/src/repository.py
index 7b1820853..baa764ad2 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -104,13 +104,14 @@ class BackupRepository(ConfigPanel):
         if not full:
             return repositories
 
+        full_repositories = {}
         for repo in repositories:
             try:
-                repositories[repo] = BackupRepository(repo).info(space_used)
-            except Exception:
-                logger.error(f"Unable to open repository {repo}")
+                full_repositories.update(BackupRepository(repo).info(space_used))
+            except Exception as e:
+                logger.error(f"Unable to open repository {repo}: {e}")
 
-        return repositories
+        return full_repositories
 
     # =================================================
     # Config Panel Hooks
@@ -232,18 +233,19 @@ class BackupRepository(ConfigPanel):
             self.purge()
 
         rm(self.save_path, force=True)
-        logger.success(m18n.n("repository_removed", repository=self.shortname))
+        logger.success(m18n.n("repository_removed", repository=self.entity))
 
     def info(self, space_used=False):
         result = super().get(mode="export")
 
-        if self.__class__ == BackupRepository and space_used is True:
+        if space_used is True:
             result["space_used"] = self.compute_space_used()
 
-        return {self.shortname: result}
+        return {self.entity: result}
 
     def list_archives(self, with_info):
-        archives = self.list_archive_name()
+        self._cast_by_method()
+        archives = self.list_archives_names()
         if with_info:
             d = {}
             for archive in archives:
@@ -267,7 +269,7 @@ class BackupRepository(ConfigPanel):
 
         # List archives with creation date
         archives = {}
-        for archive_name in self.list_archive_name(prefix):
+        for archive_name in self.list_archives_names(prefix):
             archive = BackupArchive(repo=self, name=archive_name)
             created_at = archive.info()["created_at"]
             archives[created_at] = archive
@@ -314,7 +316,7 @@ class BackupRepository(ConfigPanel):
     def purge(self):
         raise NotImplementedError()
 
-    def list_archives_names(self):
+    def list_archives_names(self, prefix=None):
         raise NotImplementedError()
 
     def compute_space_used(self):

From 3563e8dc100084c7b1e77494e7ec685428d5830a Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Sat, 15 Oct 2022 20:28:00 +0200
Subject: [PATCH 38/48] [fix] Backup list and create

---
 locales/en.json          |  13 ++--
 share/actionsmap.yml     |  21 +++++--
 src/backup.py            | 103 +++++++++++++++++++++----------
 src/repositories/borg.py |  39 +++++++-----
 src/repositories/tar.py  |  12 +++-
 src/repository.py        | 127 ++++++++++++++++++---------------------
 src/utils/filesystem.py  |   2 +
 7 files changed, 194 insertions(+), 123 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index a3aba5dd3..cbb5e24b4 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -84,14 +84,14 @@
     "backup_archive_broken_link": "Could not access the backup archive (broken link to {path})",
     "backup_archive_cant_retrieve_info_json": "Could not load info for archive '{archive}'... The info.json file cannot be retrieved (or is not a valid json).",
     "backup_archive_corrupted": "It looks like the backup archive '{archive}' is corrupted : {error}",
-    "backup_archive_name_exists": "A backup archive with this name already exists.",
+    "backup_archive_name_exists": "A backup archive with this name already exists in the repo '{repository}'.",
     "backup_archive_name_unknown": "Unknown local backup archive named '{name}'",
     "backup_archive_open_failed": "Could not open the backup archive",
     "backup_archive_system_part_not_available": "System part '{part}' unavailable in this backup",
     "backup_archive_writing_error": "Could not add the files '{source}' (named in the archive '{dest}') to be backed up into the compressed archive '{archive}'",
     "backup_ask_for_copying_if_needed": "Do you want to perform the backup using {size}MB temporarily? (This way is used since some files could not be prepared using a more efficient method.)",
     "backup_borg_init_error": "Unable initialize the borg repository: {error}",
-    "backup_borg_already_initialized": "The borg repository '{repository}' already exists, it has been properly added to repositories managed by YunoHost cli.",
+    "backup_borg_list_archive_error": "Unable to list files in the archive",
     "backup_cant_mount_uncompress_archive": "Could not mount the uncompressed archive as write protected",
     "backup_cleaning_failed": "Could not clean up the temporary backup folder",
     "backup_copying_to_organize_the_archive": "Copying {size}MB to organize the archive",
@@ -106,11 +106,14 @@
     "backup_delete_error": "Could not delete '{path}'",
     "backup_deleted": "Backup deleted",
     "backup_hook_unknown": "The backup hook '{hook}' is unknown",
-    "backup_method_copy_finished": "Backup copy finalized",
-    "backup_method_custom_finished": "Custom backup method '{method}' finished",
-    "backup_method_tar_finished": "TAR backup archive created",
+    "backuping_in_repository": "Backuping into repository '{repository}'",
+    "backup_in_repository_finished": "Backup into repository '{repository}' is finished",
+    "backup_in_repository_error": "Backup into repository '{repository}' failed: {error}",
+    "backup_invalid_archive": "Invalid backup archive : {error}",
     "backup_mount_archive_for_restore": "Preparing archive for restoration...",
     "backup_no_uncompress_archive_dir": "There is no such uncompressed archive directory",
+    "backup_not_sent": "Backup archive was not saved at all",
+    "backup_partially_sent": "Backup archive was not sent into all repositories listed",
     "backup_nothings_done": "Nothing to save",
     "backup_output_directory_forbidden": "Pick a different output directory. Backups cannot be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders",
     "backup_output_directory_not_empty": "You should pick an empty output directory",
diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index 0b3024c43..368c7e9dd 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1019,11 +1019,10 @@ backup:
             action_help: Restore from a local backup archive. If neither --apps or --system are given, this will restore all apps and all system parts in the archive. If only --apps if given, this will only restore apps and no system parts. Similarly, if only --system is given, this will only restore system parts and no apps.
             api: PUT /backups/<name>/restore
             arguments:
+                repository:
+                    help: Repository of the backup archive
                 name:
                     help: Name of the local backup archive
-                -r:
-                    full: --repository
-                    help: The archive repository (local borg repo use by default)
                 --system:
                     help: List of system parts to restore (or all if none is given)
                     nargs: "*"
@@ -1036,9 +1035,15 @@ backup:
 
         ### backup_list()
         list:
-            action_help: List available local backup archives
+            action_help: List available local backup archives or list files in an archive
             api: GET /backups
             arguments:
+                repository:
+                    help: Repository of a backup archive
+                    nargs: "?"
+                name:
+                    help: Name of a backup archive
+                    nargs: "?"
                 -r:
                     full: --repositories
                     help: List archives in these repositories
@@ -1057,8 +1062,10 @@ backup:
             action_help: Show info about a local backup archive
             api: GET /backups/<name>
             arguments:
+                repository:
+                    help: Repository of the backup archive
                 name:
-                    help: Name of the local backup archive
+                    help: Name of the backup archive
                 -d:
                     full: --with-details
                     help: Show additional backup information
@@ -1073,6 +1080,8 @@ backup:
             action_help: (API only) Request to download the file
             api: GET /backups/<name>/download
             arguments:
+                repository:
+                    help: Repository of the backup archive
                 name:
                     help: Name of the local backup archive
         
@@ -1081,6 +1090,8 @@ backup:
             action_help: Delete a backup archive
             api: DELETE /backups/<name>
             arguments:
+                repository:
+                    help: Repository of the backup archive
                 name:
                     help: Name of the archive to delete
                     extra:
diff --git a/src/backup.py b/src/backup.py
index d0bbc89d4..8797ade0a 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -273,6 +273,8 @@ class BackupManager:
         description -- (string) A description for this future backup archive
                         (default: '')
 
+        repositories-- (List<BackupRepository>) A list of repositories
+
         work_dir    -- (None|string) A path where prepare the archive. If None,
                         temporary work_dir will be created (default: None)
         """
@@ -785,13 +787,22 @@ class BackupManager:
     #
 
     def backup(self):
-        """Apply backup methods"""
-
+        """Backup files in each repository"""
+        result = {}
         for repo in self.repositories:
-            logger.debug(m18n.n("backup_applying_method_" + repo.method_name))
-            archive = BackupArchive(repo, name=self.name, manager=self)
-            archive.organize_and_backup()
-            logger.debug(m18n.n("backup_method_" + repo.method_name + "_finished"))
+            logger.debug(m18n.n("backuping_in_repository", repository=repo.entity))
+            try:
+                archive = BackupArchive(repo, name=self.name, manager=self)
+                archive.organize_and_backup()
+            except Exception:
+                import traceback
+                result[repo.entity] = "Error"
+                logger.error(m18n.n("backup_in_repository_error", repository=repo.entity, error=traceback.format_exc()))
+            else:
+                result[repo.entity] = "Sent"
+                logger.debug(m18n.n("backup_in_repository_finished", repository=repo.entity))
+
+        return result
 
     def _compute_backup_size(self):
         """
@@ -1626,9 +1637,9 @@ def backup_create(
         name -- Name of the backup archive
         description -- Short description of the backup
         repositories -- Repositories in which we want to save the backup
-        output_directory -- Output directory for the backup
         system -- List of system elements to backup
         apps -- List of application names to backup
+        dry_run -- Run ynh backup script without send the files into a repo
     """
 
     # TODO: Add a 'clean' argument to clean output directory
@@ -1637,9 +1648,19 @@ def backup_create(
     # Validate / parse arguments                                            #
     #
 
-    # Validate there is no archive with the same name
+    # Add backup repositories
 
-    if name and name in backup_list(repositories)["archives"]:
+    if not repositories:
+        repositories = ["local-borg"]
+
+    # Validate there is no archive with the same name
+    archives = backup_list(repositories=repositories)
+    for repository in archives:
+        if name and name in archives[repository]:
+            repositories.pop(repository)
+            logger.error(m18n.n("backup_archive_name_exists", repository=repository))
+
+    if not repositories:
         raise YunohostValidationError("backup_archive_name_exists")
 
     # If no --system or --apps given, backup everything
@@ -1654,14 +1675,6 @@ def backup_create(
 
     operation_logger.start()
 
-    # Create yunohost archives directory if it does not exists
-    _create_archive_dir()  # FIXME
-
-    # Add backup repositories
-
-    if not repositories:
-        repositories = ["local-borg"]
-
     repositories = [BackupRepository(repo) for repo in repositories]
 
     # Prepare files to backup
@@ -1669,7 +1682,6 @@ def backup_create(
                                    repositories=repositories)
 
     # Add backup targets (system and apps)
-
     backup_manager.set_system_targets(system)
     backup_manager.set_apps_targets(apps)
 
@@ -1684,6 +1696,12 @@ def backup_create(
     # Collect files to be backup (by calling app backup script / system hooks)
     backup_manager.collect_files()
 
+    parts_results = backup_manager.targets.results
+    parts_results = list(parts_results["apps"].values()) + list(parts_results["system"].values())
+    parts_states = [v in ["Success", "Skipped"] for v in parts_results]
+    if not any(parts_states):
+        raise YunohostError("backup_nothings_done")
+
     if dry_run:
         return {
             "size": backup_manager.size,
@@ -1698,19 +1716,36 @@ def backup_create(
             size=binary_to_human(backup_manager.size) + "B",
         )
     )
-    backup_manager.backup()
+    repo_results = backup_manager.backup()
+    repo_states = [repo_result == "Success" for repository, repo_result in repo_results.items()]
 
-    logger.success(m18n.n("backup_created"))
-    operation_logger.success()
+    if all(repo_states) and all(parts_states):
+        logger.success(m18n.n("backup_created"))
+        operation_logger.success()
+    else:
+        if not any(repo_states):
+            error = m18n.n("backup_not_sent")
+        elif not all(repo_states):
+            error = m18n.n("backup_partially_sent")
+
+        if not all(parts_states):
+            error += "\n" + m18n.n("backup_files_not_fully_collected")
+            for repository, repo_result in repo_results.items():
+                if repo_result == "Sent":
+                    repo_results[repository] = "Incomplete"
+
+        logger.error(error)
+        operation_logger.error(error)
 
     return {
         "name": backup_manager.name,
         "size": backup_manager.size,
         "results": backup_manager.targets.results,
+        "states": repo_results
     }
 
 
-def backup_restore(name, repository, system=[], apps=[], force=False):
+def backup_restore(repository, name, system=[], apps=[], force=False):
     """
     Restore from a local backup archive
 
@@ -1744,7 +1779,7 @@ def backup_restore(name, repository, system=[], apps=[], force=False):
         name = name[: -len(".tar")]
 
     repo = BackupRepository(repository)
-    archive = BackupArchive(name, repo)
+    archive = BackupArchive(repo, name)
 
     restore_manager = RestoreManager(archive)
 
@@ -1795,7 +1830,7 @@ def backup_restore(name, repository, system=[], apps=[], force=False):
     return restore_manager.targets.results
 
 
-def backup_list(repositories=[], with_info=False, human_readable=False):
+def backup_list(repository=None, name=None, repositories=[], with_info=False, human_readable=False):
     """
     List available local backup archives
 
@@ -1805,6 +1840,12 @@ def backup_list(repositories=[], with_info=False, human_readable=False):
         human_readable -- Print sizes in human readable format
 
     """
+    if bool(repository) != bool(name):
+        raise YunohostError("backup_list_bad_arguments")
+    elif repository:
+        repo = BackupRepository(repository)
+        archive = BackupArchive(repo, name)
+        return archive.list(with_info)
 
     return {
         name: BackupRepository(name).list_archives(with_info)
@@ -1813,10 +1854,10 @@ def backup_list(repositories=[], with_info=False, human_readable=False):
     }
 
 
-def backup_download(name, repository):
+def backup_download(repository, name):
 
     repo = BackupRepository(repository)
-    archive = BackupArchive(name, repo)
+    archive = BackupArchive(repo, name)
 
     return archive.download()
 
@@ -1824,12 +1865,12 @@ def backup_download(name, repository):
 def backup_mount(name, repository, path):
 
     repo = BackupRepository(repository)
-    archive = BackupArchive(name, repo)
+    archive = BackupArchive(repo, name)
 
     return archive.mount(path)
 
 
-def backup_info(name, repository=None, with_details=False, human_readable=False):
+def backup_info(repository, name, with_details=False, human_readable=False):
     """
     Get info about a local backup archive
 
@@ -1840,12 +1881,12 @@ def backup_info(name, repository=None, with_details=False, human_readable=False)
 
     """
     repo = BackupRepository(repository)
-    archive = BackupArchive(name, repo)
+    archive = BackupArchive(repo, name)
 
     return archive.info(with_details=with_details, human_readable=human_readable)
 
 
-def backup_delete(name, repository):
+def backup_delete(repository, name):
     """
     Delete a backup
 
@@ -1854,7 +1895,7 @@ def backup_delete(name, repository):
 
     """
     repo = BackupRepository(repository)
-    archive = BackupArchive(name, repo)
+    archive = BackupArchive(repo, name)
 
     # FIXME Those are really usefull ?
     hook_callback("pre_backup_delete", args=[name])
diff --git a/src/repositories/borg.py b/src/repositories/borg.py
index 894d0b7ca..a4e664b0f 100644
--- a/src/repositories/borg.py
+++ b/src/repositories/borg.py
@@ -38,7 +38,7 @@ class BorgBackupRepository(LocalBackupRepository):
     method_name = "borg"
 
     # TODO logs
-    def _run_borg_command(self, cmd, stdout=None, stderr=None):
+    def _run_borg_command(self, cmd, stdout=None, stderr=None, cwd=None):
         """ Call a submethod of borg with the good context
         """
         env = dict(os.environ)
@@ -59,13 +59,15 @@ class BorgBackupRepository(LocalBackupRepository):
 
         # Authorize to move the repository (borgbase do this)
         env["BORG_RELOCATED_REPO_ACCESS_IS_OK"] = "yes"
-
+        kwargs = {}
+        if cwd:
+            kwargs["cwd"] = cwd
         return subprocess.Popen(cmd, env=env,
-                                stdout=stdout, stderr=stderr)
+                                stdout=stdout, stderr=stderr, **kwargs)
 
-    def _call(self, action, cmd, json_output=False):
+    def _call(self, action, cmd, json_output=False, cwd=None):
         borg = self._run_borg_command(cmd, stdout=subprocess.PIPE,
-                                      stderr=subprocess.PIPE)
+                                      stderr=subprocess.PIPE, cwd=cwd)
         out, err = borg.communicate()
         if borg.returncode:
             raise YunohostError(f"backup_borg_{action}_error", error=err)
@@ -108,7 +110,7 @@ class BorgBackupRepository(LocalBackupRepository):
                 self.new_values['location'] = self.location
 
             if not self.future_values.get('user'):
-                raise YunohostError("")
+                raise YunohostError("")  # TODO
         # Local
         else:
             super().install()
@@ -132,7 +134,7 @@ class BorgBackupRepository(LocalBackupRepository):
                 except YunohostError:
                     raise e
 
-                logger.info(m18n.n("backup_borg_already_initialized", repository=self.location))
+                logger.debug("The borg repository '{self.location}' already exists.")
 
     def update(self):
         raise NotImplementedError()
@@ -213,21 +215,30 @@ class BorgBackupArchive(BackupArchive):
 
     def backup(self):
         cmd = ['borg', 'create', self.archive_path, './']
-        self.repo._call('backup', cmd)
+        self.repo._call('backup', cmd, cwd=self.work_dir)
 
     def delete(self):
         cmd = ['borg', 'delete', '--force', self.archive_path]
         self.repo._call('delete_archive', cmd)
 
-    def list(self):
+    def list(self, with_info=False):
         """ Return a list of archives names
 
         Exceptions:
         backup_borg_list_error -- Raised if the borg script failed
         """
-        cmd = ["borg", "list", "--json-lines", self.archive_path]
+        cmd = ["borg", "list", "--json-lines" if with_info else "--short",
+               self.archive_path]
         out = self.repo._call('list_archive', cmd)
-        result = [json.loads(out) for line in out.splitlines()]
+
+        if not with_info:
+            return out.decode()
+
+        result = {}
+        for line in out.splitlines():
+            _file = json.loads(line)
+            filename = _file.pop("path")
+            result[filename] = _file
         return result
 
     def download(self, exclude_paths=[]):
@@ -248,12 +259,12 @@ class BorgBackupArchive(BackupArchive):
         response.content_type = "application/x-tar"
         return HTTPResponse(reader, 200)
 
-    def extract(self, paths=None, exclude_paths=[]):
-        paths, exclude_paths = super().extract(paths, exclude_paths)
+    def extract(self, paths=None, destination=None, exclude_paths=[]):
+        paths, destination, exclude_paths = super().extract(paths, destination, exclude_paths)
         cmd = ['borg', 'extract', self.archive_path] + paths
         for path in exclude_paths:
             cmd += ['--exclude', path]
-        return self.repo._call('extract_archive', cmd)
+        return self.repo._call('extract_archive', cmd, cwd=destination)
 
     def mount(self, path):
         # FIXME How to be sure the place where we mount is secure ?
diff --git a/src/repositories/tar.py b/src/repositories/tar.py
index 9d0fe539d..f46566536 100644
--- a/src/repositories/tar.py
+++ b/src/repositories/tar.py
@@ -27,6 +27,7 @@ from moulinette.utils.log import getActionLogger
 from moulinette import m18n
 
 from yunohost.utils.error import YunohostError, YunohostValidationError
+from yunohost.utils.filesystem import free_space_in_directory
 from yunohost.repository import LocalBackupRepository
 from yunohost.backup import BackupManager
 from yunohost.utils.filesystem import space_used_in_directory
@@ -43,7 +44,13 @@ class TarBackupRepository(LocalBackupRepository):
         # (we do a realpath() to resolve symlinks)
         archives = glob(f"{self.location}/*.tar.gz") + glob(f"{self.location}/*.tar")
         archives = set([os.path.realpath(archive) for archive in archives])
-        archives = sorted(archives, key=lambda x: os.path.getctime(x))
+        broken_archives = set()
+        for archive in archives:
+            if not os.path.exists(archive):
+                broken_archives.add(archive)
+                logger.warning(m18n.n("backup_archive_broken_link", path=archive))
+
+        archives = sorted(archives - broken_archives, key=lambda x: os.path.getctime(x))
 
         # Extract only filename without the extension
         def remove_extension(f):
@@ -57,6 +64,9 @@ class TarBackupRepository(LocalBackupRepository):
     def compute_space_used(self):
         return space_used_in_directory(self.location)
 
+    def compute_free_space(self):
+        return free_space_in_directory(self.location)
+
     def prune(self):
         raise NotImplementedError()
 
diff --git a/src/repository.py b/src/repository.py
index baa764ad2..f7ee845e3 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -42,12 +42,12 @@ from datetime import timedelta, datetime
 import yunohost.repositories
 from yunohost.utils.config import ConfigPanel
 from yunohost.utils.error import YunohostError, YunohostValidationError
-from yunohost.utils.filesystem import disk_usage, binary_to_human, free_space_in_directory
+from yunohost.utils.filesystem import disk_usage, binary_to_human
 from yunohost.utils.network import get_ssh_public_key, SHF_BASE_URL
 
 logger = getActionLogger('yunohost.repository')
 REPOSITORIES_DIR = '/etc/yunohost/repositories'
-CACHE_INFO_DIR = "/var/cache/yunohost/{repository}"
+CACHE_INFO_DIR = "/var/cache/yunohost/repositories/{repository}"
 REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
 MB_ALLOWED_TO_ORGANIZE = 10
 # TODO split ConfigPanel.get to extract "Format result" part and be able to override it
@@ -113,6 +113,23 @@ class BackupRepository(ConfigPanel):
 
         return full_repositories
 
+    def __init__(self, entity, config_path=None, save_path=None, creation=False):
+
+        super().__init__(entity, config_path, save_path, creation)
+
+        self._load_current_values()
+
+        if self.__class__ == BackupRepository:
+            if self.method == 'tar':
+                from yunohost.repositories.tar import TarBackupRepository
+                self.__class__ = TarBackupRepository
+            elif self.method == 'borg':
+                from yunohost.repositories.borg import BorgBackupRepository
+                self.__class__ = BorgBackupRepository
+            else:
+                from yunohost.repositories.hook import HookBackupRepository
+                self.__class__ = HookBackupRepository
+
     # =================================================
     # Config Panel Hooks
     # =================================================
@@ -154,7 +171,6 @@ class BackupRepository(ConfigPanel):
 
         if 'shf_id' in self.values:
             self.values['is_shf'] = bool(self.values['shf_id'])
-        self._cast_by_method()
 
     def _parse_pre_answered(self, *args):
         super()._parse_pre_answered(*args)
@@ -165,7 +181,6 @@ class BackupRepository(ConfigPanel):
             self.args['method'] = "borg"
         elif self.args.get('method') == 'tar':
             self.args['is_remote'] = False
-        self._cast_by_method()
 
     def _apply(self):
         # Activate / update services
@@ -194,44 +209,34 @@ class BackupRepository(ConfigPanel):
 
         return f"ssh://{self.user}@{self.domain}:{self.port}/{self.path}"
 
-    def _cast_by_method(self):
-        if not self.future_values:
-            return
+    @property
+    def is_deduplicated(self):
+        return True
 
-        if self.__class__ == BackupRepository:
-            if self.method == 'tar':
-                from yunohost.repositories.tar import TarBackupRepository
-                self.__class__ = TarBackupRepository
-            elif self.method == 'borg':
-                from yunohost.repositories.borg import BorgBackupRepository
-                self.__class__ = BorgBackupRepository
-            else:
-                from yunohost.repositories.hook import HookBackupRepository
-                self.__class__ = HookBackupRepository
-
-    def _check_is_enough_free_space(self):
+    def check_is_enough_free_space(self, backup_size):
         """
         Check free space in repository or output directory before to backup
         """
-        # TODO How to do with distant repo or with deduplicated backup ?
-        backup_size = self.manager.size
+        if self.is_deduplicated:
+            return
 
-        free_space = free_space_in_directory(self.repo)
+        free_space = self.compute_free_space(self)
 
         if free_space < backup_size:
             logger.debug(
                 "Not enough space at %s (free: %s / needed: %d)",
-                self.repo,
+                self.entity,
                 free_space,
                 backup_size,
             )
-            raise YunohostValidationError("not_enough_disk_space", path=self.repo)
+            raise YunohostValidationError("not_enough_disk_space", path=self.entity)
 
     def remove(self, purge=False):
         if purge:
             self._load_current_values()
             self.purge()
 
+        rm(CACHE_INFO_DIR.format(repository=self.entity), recursive=True, force=True)
         rm(self.save_path, force=True)
         logger.success(m18n.n("repository_removed", repository=self.entity))
 
@@ -243,14 +248,13 @@ class BackupRepository(ConfigPanel):
 
         return {self.entity: result}
 
-    def list_archives(self, with_info):
-        self._cast_by_method()
+    def list_archives(self, with_info=False):
         archives = self.list_archives_names()
         if with_info:
             d = {}
             for archive in archives:
                 try:
-                    d[archive] = BackupArchive(repo=self, name=archive).info()
+                    d[archive] = BackupArchive(repo=self, name=archive).info(with_details=with_info)
                 except YunohostError as e:
                     logger.warning(str(e))
                 except Exception:
@@ -322,6 +326,9 @@ class BackupRepository(ConfigPanel):
     def compute_space_used(self):
         raise NotImplementedError()
 
+    def compute_free_space(self):
+        raise NotImplementedError()
+
 
 class LocalBackupRepository(BackupRepository):
     def install(self):
@@ -354,7 +361,7 @@ class BackupArchive:
             self.__class__ = yunohost.repositories.hook.HookBackupArchive
 
         # Assert archive exists
-        if self.manager.__class__.__name__ != "BackupManager" and self.name not in self.repo.list_archives():
+        if self.manager.__class__.__name__ != "BackupManager" and self.name not in self.repo.list_archives(False):
             raise YunohostValidationError("backup_archive_name_unknown", name=name)
 
     @property
@@ -377,7 +384,7 @@ class BackupArchive:
 
     # This is not a property cause it could be managed in a hook
     def need_organized_files(self):
-        return self.repo.need_organised_files
+        return self.repo.need_organized_files
 
     def organize_and_backup(self):
         """
@@ -392,7 +399,7 @@ class BackupArchive:
         self.repo.install()
 
         # Check free space in output
-        self._check_is_enough_free_space()
+        self.repo.check_is_enough_free_space(self.manager.size)
         try:
             self.backup()
         finally:
@@ -443,59 +450,45 @@ class BackupArchive:
             yield f"{leading_dot}apps/{app}"
 
     def _get_info_string(self):
-        self.archive_file = "%s/%s.tar" % (self.repo.path, self.name)
+        """Extract info file from archive if needed and read it"""
 
-        # Check file exist (even if it's a broken symlink)
-        if not os.path.lexists(self.archive_file):
-            self.archive_file += ".gz"
-            if not os.path.lexists(self.archive_file):
-                raise YunohostValidationError("backup_archive_name_unknown", name=self.name)
-
-        # If symlink, retrieve the real path
-        if os.path.islink(self.archive_file):
-            archive_file = os.path.realpath(self.archive_file)
-
-            # Raise exception if link is broken (e.g. on unmounted external storage)
-            if not os.path.exists(archive_file):
-                raise YunohostValidationError(
-                    "backup_archive_broken_link", path=archive_file
-                )
-        info_file = CACHE_INFO_DIR.format(repository=self.repo.name)
-        mkdir(info_file, mode=0o0700, parents=True, force=True)
-        info_file += f"/{self.name}.info.json"
+        cache_info_dir = CACHE_INFO_DIR.format(repository=self.repo.entity)
+        mkdir(cache_info_dir, mode=0o0700, parents=True, force=True)
+        info_file = f"{cache_info_dir}/{self.name}.info.json"
 
         if not os.path.exists(info_file):
-            info_dir = tempfile.mkdtemp()
+            tmp_dir = tempfile.mkdtemp()
             try:
                 files_in_archive = self.list()
                 if "info.json" in files_in_archive:
-                    self.extract("info.json")
+                    self.extract("info.json", destination=tmp_dir)
                 elif "./info.json" in files_in_archive:
-                    self.extract("./info.json")
+                    self.extract("./info.json", destination=tmp_dir)
                 else:
                     raise YunohostError(
-                        "backup_archive_cant_retrieve_info_json", archive=self.archive_file
+                        "backup_archive_cant_retrieve_info_json", archive=self.archive_path
                     )
-                shutil.move(os.path.join(info_dir, "info.json"), info_file)
+                    # FIXME should we cache there is no info.json ?
+                shutil.move(os.path.join(tmp_dir, "info.json"), info_file)
             finally:
-                os.rmdir(info_dir)
+                os.rmdir(tmp_dir)
 
         try:
             return read_file(info_file)
-        except MoulinetteError:
+        except MoulinetteError as e:
             logger.debug("unable to load '%s'", info_file, exc_info=1)
-            raise YunohostError('backup_invalid_archive')
+            raise YunohostError('backup_invalid_archive', error=e)
 
-    def info(self, with_details, human_readable):
+    def info(self, with_details=False, human_readable=False):
 
         info_json = self._get_info_string()
-        if not self._info_json:
+        if not info_json:
             raise YunohostError('backup_info_json_not_implemented')
         try:
-            info = json.load(info_json)
-        except Exception:
+            info = json.loads(info_json)
+        except Exception as e:
             logger.debug("unable to load info json", exc_info=1)
-            raise YunohostError('backup_invalid_archive')
+            raise YunohostError('backup_invalid_archive', error=e)
 
         # (legacy) Retrieve backup size
         # FIXME
@@ -509,7 +502,7 @@ class BackupArchive:
             )
             tar.close()
         result = {
-            "path": self.repo.archive_path,
+            "path": self.archive_path,
             "created_at": datetime.utcfromtimestamp(info["created_at"]),
             "description": info["description"],
             "size": size,
@@ -571,7 +564,7 @@ class BackupArchive:
         for path in self.manager.paths_to_backup:
             src = path["source"]
 
-            if self.manager.__class__.__name__ != "RestoreManager":
+            if self.manager.__class__.__name__ == "RestoreManager":
                 # TODO Support to run this before a restore (and not only before
                 # backup). To do that RestoreManager.unorganized_work_dir should
                 # be implemented
@@ -694,16 +687,16 @@ class BackupArchive:
             )
             return
 
-    def extract(self, paths=None, exclude_paths=[]):
+    def extract(self, paths=None, destination=None, exclude_paths=[]):
         if self.__class__ == BackupArchive:
             raise NotImplementedError()
-        if isinstance(exclude_paths, str):
+        if isinstance(paths, str):
             paths = [paths]
         elif paths is None:
             paths = self.select_files()
         if isinstance(exclude_paths, str):
             exclude_paths = [exclude_paths]
-        return paths, exclude_paths
+        return paths, destination, exclude_paths
 
     def mount(self):
         if self.__class__ == BackupArchive:
diff --git a/src/utils/filesystem.py b/src/utils/filesystem.py
index 494a0187c..f026bd767 100644
--- a/src/utils/filesystem.py
+++ b/src/utils/filesystem.py
@@ -20,6 +20,8 @@
 """
 import os
 
+from moulinette.utils.process import check_output
+
 
 def free_space_in_directory(dirpath):
     stat = os.statvfs(dirpath)

From be8216c0b4a12e8cd165f492f5091693ac8a953f Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Sun, 16 Oct 2022 15:14:52 +0200
Subject: [PATCH 39/48] [fix] Backup restore

---
 share/actionsmap.yml     |  3 ++-
 src/backup.py            | 35 +++++++++++++++++++++++------------
 src/repositories/borg.py |  2 +-
 3 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index 368c7e9dd..f1b47e279 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1092,10 +1092,11 @@ backup:
             arguments:
                 repository:
                     help: Repository of the backup archive
-                name:
+                archive_name:
                     help: Name of the archive to delete
                     extra:
                         pattern: *pattern_backup_archive_name
+                    nargs: "*"
 
     subcategories:
         repository:
diff --git a/src/backup.py b/src/backup.py
index 8797ade0a..8e6a2d64f 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -1107,7 +1107,24 @@ class RestoreManager:
 
         filesystem.mkdir(self.work_dir, parents=True)
 
-        self.archive.extract()
+        # Select paths to extract
+        paths = ["backup.csv", "info.json", "hooks"]
+        paths += [f"apps/{app}" for app in self.targets.list("apps", exclude=["Skipped"])]
+        for system in self.targets.list("system", exclude=["Skipped"]):
+            if system.startswith("data"):
+                paths.append(f"data/{system}")
+            elif system.startswith("conf_ynh"):
+                if "conf/ynh" not in paths:
+                    paths.append("conf/ynh")
+            else:
+                paths.append(system.replace("_", "/", 1))
+
+        if not self.targets.list("system", exclude=["Skipped"]):
+            paths.remove("hooks")
+
+        logger.debug(f"List of paths to extract: {paths}")
+
+        self.archive.extract(paths=paths, destination=self.work_dir)
 
     #
     # Space computation / checks                                            #
@@ -1717,7 +1734,7 @@ def backup_create(
         )
     )
     repo_results = backup_manager.backup()
-    repo_states = [repo_result == "Success" for repository, repo_result in repo_results.items()]
+    repo_states = [repo_result == "Sent" for repository, repo_result in repo_results.items()]
 
     if all(repo_states) and all(parts_states):
         logger.success(m18n.n("backup_created"))
@@ -1886,7 +1903,7 @@ def backup_info(repository, name, with_details=False, human_readable=False):
     return archive.info(with_details=with_details, human_readable=human_readable)
 
 
-def backup_delete(repository, name):
+def backup_delete(repository, archive_name):
     """
     Delete a backup
 
@@ -1894,15 +1911,9 @@ def backup_delete(repository, name):
         name -- Name of the local backup archive
 
     """
-    repo = BackupRepository(repository)
-    archive = BackupArchive(repo, name)
-
-    # FIXME Those are really usefull ?
-    hook_callback("pre_backup_delete", args=[name])
-
-    archive.delete()
-
-    hook_callback("post_backup_delete", args=[name])
+    for name in archive_name:
+        repo = BackupRepository(repository)
+        BackupArchive(repo, name).delete()
 
     logger.success(m18n.n("backup_deleted"))
 
diff --git a/src/repositories/borg.py b/src/repositories/borg.py
index a4e664b0f..934010d44 100644
--- a/src/repositories/borg.py
+++ b/src/repositories/borg.py
@@ -259,7 +259,7 @@ class BorgBackupArchive(BackupArchive):
         response.content_type = "application/x-tar"
         return HTTPResponse(reader, 200)
 
-    def extract(self, paths=None, destination=None, exclude_paths=[]):
+    def extract(self, paths=[], destination=None, exclude_paths=[]):
         paths, destination, exclude_paths = super().extract(paths, destination, exclude_paths)
         cmd = ['borg', 'extract', self.archive_path] + paths
         for path in exclude_paths:

From 90de6fdc8091b80d7cb6e2da6bea79dd1d423a5a Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Sun, 16 Oct 2022 15:49:24 +0200
Subject: [PATCH 40/48] [enh] Backup mount

---
 locales/en.json      |  1 +
 share/actionsmap.yml | 12 ++++++++++++
 src/backup.py        |  4 ++--
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index cbb5e24b4..4fdf13363 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -92,6 +92,7 @@
     "backup_ask_for_copying_if_needed": "Do you want to perform the backup using {size}MB temporarily? (This way is used since some files could not be prepared using a more efficient method.)",
     "backup_borg_init_error": "Unable initialize the borg repository: {error}",
     "backup_borg_list_archive_error": "Unable to list files in the archive",
+    "backup_borg_mount_archive_error": "Unable to mount the archive here: {error}",
     "backup_cant_mount_uncompress_archive": "Could not mount the uncompressed archive as write protected",
     "backup_cleaning_failed": "Could not clean up the temporary backup folder",
     "backup_copying_to_organize_the_archive": "Copying {size}MB to organize the archive",
diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index f1b47e279..a269c8809 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1085,6 +1085,18 @@ backup:
                 name:
                     help: Name of the local backup archive
         
+        ### backup_mount()
+        mount:
+            action_help: Mount a backup archive if possible
+            api: DELETE /backups/<name>
+            arguments:
+                repository:
+                    help: Repository of the backup archive
+                name:
+                    help: Name of the backup archive
+                path:
+                    help: Path where mount the archive
+        
         ### backup_delete()
         delete:
             action_help: Delete a backup archive
diff --git a/src/backup.py b/src/backup.py
index 8e6a2d64f..cb73caf71 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -1879,12 +1879,12 @@ def backup_download(repository, name):
     return archive.download()
 
 
-def backup_mount(name, repository, path):
+def backup_mount(repository, name, path):
 
     repo = BackupRepository(repository)
     archive = BackupArchive(repo, name)
 
-    return archive.mount(path)
+    archive.mount(path)
 
 
 def backup_info(repository, name, with_details=False, human_readable=False):

From c73a677d20e1ea062730b3dcd238f36bda7c6488 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Sun, 16 Oct 2022 19:19:57 +0200
Subject: [PATCH 41/48] [fix] Relative path question

---
 src/utils/config.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/utils/config.py b/src/utils/config.py
index e767816a8..705c399c2 100644
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -1017,8 +1017,10 @@ class PathQuestion(Question):
                     name=option.get("name"),
                     error="Question is mandatory",
                 )
-
-        return "/" + value.strip().strip(" /")
+        value = value.strip().strip(" /")
+        if not value.startswith("~"):
+            value = "/" + value
+        return value
 
 
 class BooleanQuestion(Question):

From ac0b4c43024613ec8e263d54a92e05f47b9e9218 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Sun, 16 Oct 2022 19:20:50 +0200
Subject: [PATCH 42/48] [fix] Add remote repo

---
 src/repositories/borg.py | 11 +++++++++--
 src/repository.py        | 17 ++++++++++++++++-
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/repositories/borg.py b/src/repositories/borg.py
index 934010d44..295eeeb16 100644
--- a/src/repositories/borg.py
+++ b/src/repositories/borg.py
@@ -45,11 +45,15 @@ class BorgBackupRepository(LocalBackupRepository):
 
         if self.domain:
             # TODO Use the best/good key
-            private_key = "/root/.ssh/ssh_host_ed25519_key"
+            private_key = "/etc/ssh/ssh_host_ed25519_key"
 
             # Don't check ssh fingerprint strictly the first time
             # TODO improve this by publishing and checking this with DNS
-            strict = 'yes' if self.domain in open('/root/.ssh/known_hosts').read() else 'no'
+            # FIXME known_host are hashed now
+            try:
+                strict = 'yes' if self.domain in open('/root/.ssh/known_hosts').read() else 'no'
+            except FileNotFoundError:
+                strict = 'no'
             env['BORG_RSH'] = "ssh -i %s -oStrictHostKeyChecking=%s"
             env['BORG_RSH'] = env['BORG_RSH'] % (private_key, strict)
 
@@ -121,6 +125,8 @@ class BorgBackupRepository(LocalBackupRepository):
 
         if "quota" in self.future_values and self.future_values["quota"]:
             cmd += ['--storage-quota', self.quota]
+
+        logger.debug(cmd)
         try:
             self._call('init', cmd)
         except YunohostError as e:
@@ -260,6 +266,7 @@ class BorgBackupArchive(BackupArchive):
         return HTTPResponse(reader, 200)
 
     def extract(self, paths=[], destination=None, exclude_paths=[]):
+        # TODO exclude_paths not available in actions map
         paths, destination, exclude_paths = super().extract(paths, destination, exclude_paths)
         cmd = ['borg', 'extract', self.archive_path] + paths
         for path in exclude_paths:
diff --git a/src/repository.py b/src/repository.py
index f7ee845e3..a28ddf5e0 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -119,7 +119,10 @@ class BackupRepository(ConfigPanel):
 
         self._load_current_values()
 
-        if self.__class__ == BackupRepository:
+        self._cast_by_backup_method()
+
+    def _cast_by_backup_method(self):
+        try:
             if self.method == 'tar':
                 from yunohost.repositories.tar import TarBackupRepository
                 self.__class__ = TarBackupRepository
@@ -129,6 +132,8 @@ class BackupRepository(ConfigPanel):
             else:
                 from yunohost.repositories.hook import HookBackupRepository
                 self.__class__ = HookBackupRepository
+        except KeyError:
+            pass
 
     # =================================================
     # Config Panel Hooks
@@ -150,6 +155,16 @@ class BackupRepository(ConfigPanel):
         logger.debug("SHF running")
         return {'is_shf': True}
 
+    def post_ask__is_remote(self, question):
+        if question.value:
+            self.method = 'borg'
+        self._cast_by_backup_method()
+        return {}
+
+    def post_ask__method(self, question):
+        self._cast_by_backup_method()
+        return {}
+
     # =================================================
     # Config Panel Override
     # =================================================

From fb62a9684bb8494a008b9cda58daf446ae2a1676 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 17 Oct 2022 01:20:20 +0200
Subject: [PATCH 43/48] [fix] Backup remote restore

---
 src/backup.py       | 4 +---
 src/repository.py   | 5 ++---
 src/utils/system.py | 8 ++++++++
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/backup.py b/src/backup.py
index 47522e7e7..ee1fd69b1 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -29,7 +29,7 @@ from packaging import version
 
 from moulinette import Moulinette, m18n
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml, rm, chown, chmod, write_to_file
+from moulinette.utils.filesystem import mkdir, write_to_yaml, read_yaml, rm, chown, chmod, write_to_file
 from moulinette.utils.process import check_output
 
 import yunohost.domain
@@ -58,8 +58,6 @@ from yunohost.log import OperationLogger, is_unit_operation
 from yunohost.repository import BackupRepository, BackupArchive
 from yunohost.utils.config import ConfigPanel
 from yunohost.utils.error import YunohostError, YunohostValidationError
-from yunohost.utils.packages import ynh_packages_version
-from yunohost.utils.filesystem import free_space_in_directory, disk_usage, binary_to_human
 from yunohost.utils.system import (
     free_space_in_directory,
     get_ynh_package_version,
diff --git a/src/repository.py b/src/repository.py
index a28ddf5e0..6f50935a6 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -42,7 +42,7 @@ from datetime import timedelta, datetime
 import yunohost.repositories
 from yunohost.utils.config import ConfigPanel
 from yunohost.utils.error import YunohostError, YunohostValidationError
-from yunohost.utils.filesystem import disk_usage, binary_to_human
+from yunohost.utils.system import disk_usage, binary_to_human
 from yunohost.utils.network import get_ssh_public_key, SHF_BASE_URL
 
 logger = getActionLogger('yunohost.repository')
@@ -184,8 +184,7 @@ class BackupRepository(ConfigPanel):
         if self.values.get('method') == 'tar' and self.values['is_remote']:
             raise YunohostError("repository_tar_only_local")
 
-        if 'shf_id' in self.values:
-            self.values['is_shf'] = bool(self.values['shf_id'])
+        self.values['is_shf'] = bool(self.values['shf_id']) if 'shf_id' in self.values else False
 
     def _parse_pre_answered(self, *args):
         super()._parse_pre_answered(*args)
diff --git a/src/utils/system.py b/src/utils/system.py
index 63f7190f8..b04101d62 100644
--- a/src/utils/system.py
+++ b/src/utils/system.py
@@ -58,6 +58,14 @@ def space_used_by_directory(dirpath, follow_symlinks=True):
     return stat.f_frsize * stat.f_blocks  # FIXME : this doesnt do what the function name suggest this does ...
 
 
+def disk_usage(path):
+    # We don't do this in python with os.stat because we don't want
+    # to follow symlinks
+
+    du_output = check_output(["du", "-sb", path], shell=False)
+    return int(du_output.split()[0])
+
+
 def human_to_binary(size: str) -> int:
 
     symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")

From 97ce3f188b68e91ba2ce2f45a2e7cbc60133ba34 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 17 Oct 2022 02:39:26 +0200
Subject: [PATCH 44/48] [fix] Backup timer add

---
 share/actionsmap.yml           |  9 +---
 share/config_backup_timer.toml | 96 +++++++++++++---------------------
 src/backup.py                  | 31 ++++++++---
 src/repositories/tar.py        |  4 ++
 4 files changed, 66 insertions(+), 74 deletions(-)

diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index a82af0799..d3329d65d 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1272,11 +1272,6 @@ backup:
                 list:
                     action_help: List backup timer
                     api: GET /backup/timer
-                    arguments:
-                        -r:
-                            full: --repositories
-                            help: List archives in these repositories
-                            nargs: "*"
 
                 ### backup_timer_add()
                 add:
@@ -1307,13 +1302,13 @@ backup:
                         --alert:
                             help: Email to alert
                         --keep-hourly:
-                            default: 2
+                            default: 0
                         --keep-daily:
                             default: 7
                         --keep-weekly:
                             default: 8
                         --keep-monthly:
-                            default: 12
+                            default: 8
 
                 ### backup_timer_update()
                 update:
diff --git a/share/config_backup_timer.toml b/share/config_backup_timer.toml
index cc0c5290f..479cc255a 100644
--- a/share/config_backup_timer.toml
+++ b/share/config_backup_timer.toml
@@ -1,6 +1,6 @@
 
 version = "1.0"
-i18n = "repository_config"
+i18n = "backup_timer_config"
 [main]
 name.en = ""
     [main.main]
@@ -11,79 +11,53 @@ name.en = ""
         type = "string"
         default = ""
 
-        [main.main.is_remote]
-        type = "boolean"
-        yes = true
-        no = false
+        [main.main.repositories]
+        type = "tags"
         visible = "creation"
         default = "no"
-
-        [main.main.domain]
+        
+        [main.main.system]
+        type = "tags"
+        default = []
+        
+        [main.main.apps]
+        type = "tags"
+        default = []
+        
+        [main.main.schedule]
         type = "string"
-        visible = "creation && is_remote"
-        pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
-        pattern.error = 'domain_error' # TODO "Please provide a valid domain"
-        default = ""
-        # FIXME: can't be a domain of this instances ?
-
-        [main.main.is_shf]
-        help = ""
-        type = "boolean"
-        yes = true
-        no = false
-        visible = "creation && is_remote"
-        default = false
-
-        [main.main.public_key]
-        type = "alert"
-        style = "info"
-        visible = "creation && is_remote && ! is_shf"
+        default = "Daily"
 
         [main.main.alert]
         help = ''
         type = "tags"
-        visible = "is_remote && is_shf"
         pattern.regexp = '^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$'
         pattern.error = "alert_error"
         default = []
         # "value": alert,
 
-        [main.main.alert_delay]
+        [main.main.keep_hourly]
         help = ''
         type = "number"
-        visible = "is_remote && is_shf"
-        min = 1
-        default = 7
-
-        [main.main.quota]
-        type = "string"
-        visible = "is_remote && is_shf"
-        pattern.regexp = '^\d+[MGT]$'
-        pattern.error = '' # TODO ""
-        default = ""
-
-        [main.main.port]
-        type = "number"
-        visible = "is_remote && !is_shf"
-        min = 1
-        max = 65535
-        default = 22
-
-        [main.main.user]
-        type = "string"
-        visible = "is_remote && !is_shf"
-        default = ""
-
-        [main.main.method]
-        type = "select"
-        # "value": method,
-        choices.borg = "BorgBackup (recommended)"
-        choices.tar = "Legacy tar archive mechanism"
-        default = "borg"
-        visible = "!is_remote"
+        min = 0
+        default = 0
         
-        [main.main.path]
-        type = "path"
-        visible = "!is_remote or (is_remote and !is_shf)"
-        default = "/home/yunohost.backup/archives"
+        [main.main.keep_daily]
+        help = ''
+        type = "number"
+        min = 0
+        default = 10
+        
+        [main.main.keep_weekly]
+        help = ''
+        type = "number"
+        min = 0
+        default = 8
+        
+        [main.main.keep_monthly]
+        help = ''
+        type = "number"
+        min = 0
+        default = 8
+
 
diff --git a/src/backup.py b/src/backup.py
index ee1fd69b1..237575f21 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -2003,7 +2003,7 @@ class BackupTimer(ConfigPanel):
         if os.path.exists(self.service_path) and os.path.isfile(self.service_path):
             raise NotImplementedError()  # TODO
 
-    def _apply(self, values):
+    def _apply(self):
         write_to_file(self.save_path, f"""[Unit]
 Description=Run backup {self.entity} regularly
 
@@ -2025,6 +2025,24 @@ ExecStart=/usr/bin/yunohost backup create -n '{self.entity}' -r '{self.repositor
 User=root
 Group=root
 """)
+    @classmethod
+    def list(cls, full=False):
+        """
+        List backup timer
+        """
+        timers = super().list()
+
+        if not full:
+            return timers
+
+        full_timers = {}
+        for timer in timers:
+            try:
+                full_timers.update(BackupTimer(timer).info())
+            except Exception as e:
+                logger.error(f"Unable to open timer {timer}: {e}")
+
+        return full_timers
 
 
 def backup_timer_list(full=False):
@@ -2034,19 +2052,20 @@ def backup_timer_list(full=False):
     return {"backup_timer": BackupTimer.list(full)}
 
 
-def backup_timer_info(shortname, space_used=False):
-    return BackupTimer(shortname).info(space_used)
+def backup_timer_info(name):
+    return BackupTimer(name).get()
 
 
 @is_unit_operation()
 def backup_timer_add(
     operation_logger,
-    shortname=None,
+    name=None,
     description=None,
-    repos=[],
+    repositories=[],
     system=[],
     apps=[],
     schedule=None,
+    alert=[],
     keep_hourly=None,
     keep_daily=None,
     keep_weekly=None,
@@ -2057,7 +2076,7 @@ def backup_timer_add(
     Add a backup timer
     """
     args = {k: v for k, v in locals().items() if v is not None}
-    timer = BackupTimer(shortname, creation=True)
+    timer = BackupTimer(name, creation=True)
     return timer.set(
         operation_logger=args.pop('operation_logger'),
         args=urllib.parse.urlencode(args)
diff --git a/src/repositories/tar.py b/src/repositories/tar.py
index f46566536..f6dd7c986 100644
--- a/src/repositories/tar.py
+++ b/src/repositories/tar.py
@@ -39,6 +39,10 @@ class TarBackupRepository(LocalBackupRepository):
     need_organized_files = False
     method_name = "tar"
 
+    # =================================================
+    # Repository actions
+    # =================================================
+
     def list_archives_names(self):
         # Get local archives sorted according to last modification time
         # (we do a realpath() to resolve symlinks)

From ad5c0f0cc624e34e91a59af56eafe88e087d0d64 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 18 Oct 2022 00:53:13 +0200
Subject: [PATCH 45/48] [fix] Backup timer

---
 share/actionsmap.yml |  27 +++++++
 src/backup.py        | 185 +++++++++++++++++++++++++++++++++----------
 src/repository.py    |   2 +-
 3 files changed, 171 insertions(+), 43 deletions(-)

diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index d3329d65d..415c3ce49 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1264,6 +1264,19 @@ backup:
                         --purge:
                             help: Remove all archives and data inside repository
                             action: store_true
+                
+                ### backup_repository_prune()
+                prune:
+                    action_help: Prune archives in a backup repository
+                    api: POST /backups/repository/<shortname>/prune
+                    arguments:
+                        shortname:
+                            help: Name of the backup repository to prune
+                            extra:
+                                pattern: *pattern_backup_repository_shortname
+                        --prefix:
+                            help: Prefix on which we prune
+                            nargs: "?"
         timer:
             subcategory_help: Manage backup timer
             actions:
@@ -1272,6 +1285,10 @@ backup:
                 list:
                     action_help: List backup timer
                     api: GET /backup/timer
+                    arguments:
+                        --full:
+                            help: Show more details
+                            action: store_true
 
                 ### backup_timer_add()
                 add:
@@ -1365,6 +1382,16 @@ backup:
                             extra:
                                 pattern: *pattern_backup_timer_name
 
+                ### backup_timer_run()
+                run:
+                    action_help: Run a backup timer
+                    api: POST /backup/timer/<name>/run
+                    arguments:
+                        name:
+                            help: Backup timer to run
+                            extra:
+                                pattern: *pattern_backup_timer_name
+
 #############################
 #         Settings          #
 #############################
diff --git a/src/backup.py b/src/backup.py
index 237575f21..74bedff52 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -1954,7 +1954,7 @@ def backup_repository_add(operation_logger, shortname, name=None, location=None,
 
     return repository.set(
         operation_logger=args.pop('operation_logger'),
-        args=urllib.parse.urlencode(args)
+        args=urllib.parse.urlencode(args, doseq=True)
     )
 
 
@@ -1977,6 +1977,20 @@ def backup_repository_remove(operation_logger, shortname, purge=False):
     BackupRepository(shortname).remove(purge)
 
 
+@is_unit_operation()
+def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=0, keep_daily=10, keep_weekly=8, keep_monthly=8):
+    """
+    Remove a backup repository
+    """
+    BackupRepository(shortname).prune(
+        prefix=prefix,
+        keep_hourly=keep_hourly,
+        keep_daily=keep_daily,
+        keep_weekly=keep_weekly,
+        keep_monthly=keep_monthly,
+    )
+
+
 #
 # Timer subcategory
 #
@@ -1986,45 +2000,15 @@ class BackupTimer(ConfigPanel):
     BackupRepository manage all repository the admin added to the instance
     """
     entity_type = "backup_timer"
-    save_path_tpl = "/etc/systemd/system/backup_timer_{entity}.timer"
+    timer_name_tpl = "backup_{entity}"
+    save_path_tpl = "/etc/yunohost/backup/timer/{entity}.yml"
+    timer_path_tpl = "/etc/systemd/system/{timer_name}.timer"
+    service_path_tpl = "/etc/systemd/system/{timer_name}.service"
     save_mode = "full"
 
-    @property
-    def service_path(self):
-        return self.save_path[:-len(".timer")] + ".service"
+    # TODO prefill apps and system question with good values
+    # TODO validate calendar entry
 
-    def _load_current_values(self):
-        # Inject defaults if needed (using the magic .update() ;))
-        self.values = self._get_default_values()
-
-        if os.path.exists(self.save_path) and os.path.isfile(self.save_path):
-            raise NotImplementedError()  # TODO
-
-        if os.path.exists(self.service_path) and os.path.isfile(self.service_path):
-            raise NotImplementedError()  # TODO
-
-    def _apply(self):
-        write_to_file(self.save_path, f"""[Unit]
-Description=Run backup {self.entity} regularly
-
-[Timer]
-OnCalendar={self.values['schedule']}
-
-[Install]
-WantedBy=timers.target
-""")
-        # TODO --system and --apps params
-        # TODO prune params
-        write_to_file(self.service_path, f"""[Unit]
-Description=Run backup {self.entity}
-After=network.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/bin/yunohost backup create -n '{self.entity}' -r '{self.repositories}' --system --apps ; /usr/bin/yunohost backup prune -n '{self.entity}'
-User=root
-Group=root
-""")
     @classmethod
     def list(cls, full=False):
         """
@@ -2038,12 +2022,121 @@ Group=root
         full_timers = {}
         for timer in timers:
             try:
-                full_timers.update(BackupTimer(timer).info())
+                full_timers[timer] = BackupTimer(timer).info()
             except Exception as e:
                 logger.error(f"Unable to open timer {timer}: {e}")
 
         return full_timers
 
+    @property
+    def timer_name(self):
+        return self.timer_name_tpl.format(entity=self.entity)
+
+    @property
+    def service_path(self):
+        return self.service_path_tpl.format(timer_name=self.timer_name)
+
+    @property
+    def timer_path(self):
+        return self.timer_path_tpl.format(timer_name=self.timer_name)
+
+    def _reload_systemd(self):
+        try:
+            check_output("systemctl daemon-reload")
+        except Exception as e:
+            logger.warning(f"Failed to reload daemon : {e}")
+
+    def _run_service_command(self, action, *args):
+        # TODO improve services to support timers
+        # See https://github.com/YunoHost/issues/issues/1519
+        try:
+            check_output(f"systemctl {action} {self.timer_name}.timer")
+        except Exception as e:
+            logger.warning(f"Failed to {action} {self.timer_name}.timer : {e}")
+
+    def _load_current_values(self):
+        super()._load_current_values()
+
+        # Search OnCalendar schedule property
+        if os.path.exists(self.timer_path) and os.path.isfile(self.timer_path):
+            with open(self.timer_path, 'r') as f:
+                for index, line in enumerate(f):
+                    if line.startswith("OnCalendar="):
+                        self.values["schedule"] = line[11:].strip()
+                        break
+                else:
+                    logger.debug(f"No OnCalendar property found in {self.timer_path}")
+
+    def _apply(self):
+
+        super()._apply()
+
+        # TODO Add RandomizedDelaySec for daily and other special event
+        write_to_file(self.timer_path, f"""[Unit]
+Description=Run backup {self.entity} regularly
+
+[Timer]
+OnCalendar={self.values['schedule']}
+
+[Install]
+WantedBy=timers.target
+""")
+        write_to_file(self.service_path, f"""[Unit]
+Description=Run backup {self.entity}
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/yunohost backup timer run '{self.entity}'
+User=root
+Group=root
+""")
+        self._reload_systemd()
+        self._run_service_command("reset-failed")
+        self.start()
+
+    def info(self):
+        return self.get(mode="export")
+
+    def remove(self):
+
+        self.stop()
+        rm(self.save_path, force=True)
+        rm(self.service_path, force=True)
+        rm(self.timer_path, force=True)
+        self._reload_systemd()
+        self._run_service_command("reset-failed")
+        logger.success(m18n.n("backup_timer_removed", timer=self.entity))
+
+    def start(self):
+        self._run_service_command("enable")
+        self._run_service_command("start")
+
+    def stop(self):
+        self._run_service_command("stop")
+        self._run_service_command("disable")
+
+    def run(self, operation_logger):
+        self._load_current_values()
+        backup_create(
+            operation_logger,
+            name=self.entity,
+            description=self.description,
+            repositories=self.repositories,
+            system=self.system,
+            apps=self.apps
+        )
+        for repository in self.repositories:
+            backup_repository_prune(
+                operation_logger,
+                shortname=repository,
+                prefix=self.entity,
+                keep_hourly=self.keep_hourly,
+                keep_daily=self.keep_daily,
+                keep_weekly=self.keep_weekly,
+                keep_monthly=self.keep_monthly,
+            )
+
 
 def backup_timer_list(full=False):
     """
@@ -2053,7 +2146,7 @@ def backup_timer_list(full=False):
 
 
 def backup_timer_info(name):
-    return BackupTimer(name).get()
+    return BackupTimer(name).info()
 
 
 @is_unit_operation()
@@ -2079,7 +2172,7 @@ def backup_timer_add(
     timer = BackupTimer(name, creation=True)
     return timer.set(
         operation_logger=args.pop('operation_logger'),
-        args=urllib.parse.urlencode(args)
+        args=urllib.parse.urlencode(args, doseq=True)
     )
 
 
@@ -2095,11 +2188,19 @@ def backup_timer_update(operation_logger, shortname, name=None,
 
 
 @is_unit_operation()
-def backup_timer_remove(operation_logger, shortname, purge=False):
+def backup_timer_remove(operation_logger, name):
     """
     Remove a backup timer
     """
-    BackupTimer(shortname).remove(purge)
+    BackupTimer(name).remove()
+
+
+@is_unit_operation()
+def backup_timer_run(operation_logger, name):
+    """
+    Run a backup timer
+    """
+    BackupTimer(name).run(operation_logger)
 
 
 #
diff --git a/src/repository.py b/src/repository.py
index 6f50935a6..653043dd6 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -46,7 +46,7 @@ from yunohost.utils.system import disk_usage, binary_to_human
 from yunohost.utils.network import get_ssh_public_key, SHF_BASE_URL
 
 logger = getActionLogger('yunohost.repository')
-REPOSITORIES_DIR = '/etc/yunohost/repositories'
+REPOSITORIES_DIR = '/etc/yunohost/backup/repositories'
 CACHE_INFO_DIR = "/var/cache/yunohost/repositories/{repository}"
 REPOSITORY_CONFIG_PATH = "/usr/share/yunohost/other/config_repository.toml"
 MB_ALLOWED_TO_ORGANIZE = 10

From 1cdbeef17f3a6e251f6e5eb114f5c1c303ffe538 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 24 Oct 2022 16:13:19 +0200
Subject: [PATCH 46/48] [fix] Store tags as list if default is a list

---
 src/utils/config.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/utils/config.py b/src/utils/config.py
index 93a0675e9..77842a363 100644
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -1080,10 +1080,17 @@ class TagsQuestion(Question):
 
     @staticmethod
     def normalize(value, option={}):
-        if isinstance(value, list):
+        option = option.__dict__ if isinstance(option, Question) else option
+
+        list_mode = "default" in option and isinstance(option["default"], list)
+
+        if isinstance(value, list) and not list_mode:
             return ",".join(value)
+
         if isinstance(value, str):
             value = value.strip()
+            if list_mode:
+                value = value.split(",")
         return value
 
     def _prevalidate(self):
@@ -1098,7 +1105,7 @@ class TagsQuestion(Question):
         self.value = values
 
     def _post_parse_value(self):
-        if isinstance(self.value, list):
+        if isinstance(self.value, list) and not isinstance(self.default, list):
             self.value = ",".join(self.value)
         return super()._post_parse_value()
 

From 5d9b91af965cfc166a19b87b84c0e0b077c7516e Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 24 Oct 2022 16:14:27 +0200
Subject: [PATCH 47/48] [enh] Pause and start a timer

---
 share/actionsmap.yml           | 27 ++++++++++++++++++++-------
 share/config_backup_timer.toml |  2 +-
 src/backup.py                  | 22 +++++++++++++++-------
 3 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index 415c3ce49..062ec782c 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1290,8 +1290,8 @@ backup:
                             help: Show more details
                             action: store_true
 
-                ### backup_timer_add()
-                add:
+                ### backup_timer_create()
+                create:
                     action_help: Add a backup timer
                     api: POST /backup/timer/<name>
                     arguments:
@@ -1382,13 +1382,26 @@ backup:
                             extra:
                                 pattern: *pattern_backup_timer_name
 
-                ### backup_timer_run()
-                run:
-                    action_help: Run a backup timer
-                    api: POST /backup/timer/<name>/run
+                ### backup_timer_start()
+                start:
+                    action_help: Start a backup timer
+                    api: POST /backup/timer/<name>/start
                     arguments:
                         name:
-                            help: Backup timer to run
+                            help: Backup timer to start
+                            extra:
+                                pattern: *pattern_backup_timer_name
+                        --now:
+                            help: Trigger a backup immediately
+                            action: store_true
+
+                ### backup_timer_pause()
+                pause:
+                    action_help: Pause a backup timer
+                    api: POST /backup/timer/<name>/pause
+                    arguments:
+                        name:
+                            help: Backup timer to pause
                             extra:
                                 pattern: *pattern_backup_timer_name
 
diff --git a/share/config_backup_timer.toml b/share/config_backup_timer.toml
index 479cc255a..27720bec5 100644
--- a/share/config_backup_timer.toml
+++ b/share/config_backup_timer.toml
@@ -14,7 +14,7 @@ name.en = ""
         [main.main.repositories]
         type = "tags"
         visible = "creation"
-        default = "no"
+        default = []
         
         [main.main.system]
         type = "tags"
diff --git a/src/backup.py b/src/backup.py
index 74bedff52..e394c5a42 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -2116,10 +2116,9 @@ Group=root
         self._run_service_command("stop")
         self._run_service_command("disable")
 
-    def run(self, operation_logger):
+    def run(self):
         self._load_current_values()
         backup_create(
-            operation_logger,
             name=self.entity,
             description=self.description,
             repositories=self.repositories,
@@ -2128,7 +2127,6 @@ Group=root
         )
         for repository in self.repositories:
             backup_repository_prune(
-                operation_logger,
                 shortname=repository,
                 prefix=self.entity,
                 keep_hourly=self.keep_hourly,
@@ -2150,7 +2148,7 @@ def backup_timer_info(name):
 
 
 @is_unit_operation()
-def backup_timer_add(
+def backup_timer_create(
     operation_logger,
     name=None,
     description=None,
@@ -2196,11 +2194,21 @@ def backup_timer_remove(operation_logger, name):
 
 
 @is_unit_operation()
-def backup_timer_run(operation_logger, name):
+def backup_timer_start(operation_logger, name, now=False):
     """
-    Run a backup timer
+    Start a backup timer
     """
-    BackupTimer(name).run(operation_logger)
+    if now:
+        BackupTimer(name).run()
+
+    BackupTimer(name).start()
+
+@is_unit_operation()
+def backup_timer_pause(operation_logger, name):
+    """
+    Pause a backup timer
+    """
+    BackupTimer(name).stop()
 
 
 #

From 3a6f1bd6127865c626829f9af6031ab888555fe9 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 24 Oct 2022 23:18:32 +0200
Subject: [PATCH 48/48] [enh] Prune and keep options

---
 locales/en.json          |  1 +
 share/actionsmap.yml     | 34 ++++++++++++++++++++++++++
 src/backup.py            | 27 +++++++++++++-------
 src/repositories/borg.py | 39 -----------------------------
 src/repository.py        | 53 +++++++++++++++++++++++++++++++---------
 5 files changed, 95 insertions(+), 59 deletions(-)

diff --git a/locales/en.json b/locales/en.json
index 8a8ef457c..3121c3e51 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -114,6 +114,7 @@
     "backup_not_sent": "Backup archive was not saved at all",
     "backup_partially_sent": "Backup archive was not sent into all repositories listed",
     "backup_nothings_done": "Nothing to save",
+    "backup_nowhere_to_backup": "Nowhere to backup your file with this archive name",
     "backup_output_directory_forbidden": "Pick a different output directory. Backups cannot be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders",
     "backup_output_directory_not_empty": "You should pick an empty output directory",
     "backup_output_directory_required": "You must provide an output directory for the backup",
diff --git a/share/actionsmap.yml b/share/actionsmap.yml
index 062ec782c..9fbfab049 100644
--- a/share/actionsmap.yml
+++ b/share/actionsmap.yml
@@ -1047,6 +1047,13 @@ backup:
                         pattern: &pattern_backup_archive_name
                             - !!str ^[\w\-\._]{1,50}$
                             - "pattern_backup_archive_name"
+                -p:
+                    full: --prefix
+                    help: Prefix of the backup archive
+                    extra:
+                        pattern: &pattern_backup_archive_prefix
+                            - !!str ^[\w\-\._]{1,35}$
+                            - "pattern_backup_archive_prefix"
                 -d:
                     full: --description
                     help: Short description of the backup
@@ -1277,6 +1284,33 @@ backup:
                         --prefix:
                             help: Prefix on which we prune
                             nargs: "?"
+                        -H:
+                            full: --keep-hourly
+                            help: Number of hourly archives to keep
+                            type: int
+                        -d:
+                            full: --keep-daily
+                            help: Number of daily archives to keep
+                            type: int
+                        -w:
+                            full: --keep-weekly
+                            help: Number of weekly archives to keep
+                            type: int
+                        -m:
+                            full: --keep-monthly
+                            help: Number of monthly archives to keep
+                            type: int
+                        --keep-last:
+                            help: Number of last archives to keep
+                            type: int
+                        --keep-within:
+                            help: Keep all archives within this time interval
+                            extra:
+                                pattern: &pattern_interval
+                                    - !!str ^\d+[Hdwmy]$
+                                    - "pattern_interval"
+
+
         timer:
             subcategory_help: Manage backup timer
             actions:
diff --git a/src/backup.py b/src/backup.py
index e394c5a42..bf3abaeca 100644
--- a/src/backup.py
+++ b/src/backup.py
@@ -25,6 +25,7 @@ import csv
 import tempfile
 import re
 import urllib.parse
+import datetime
 from packaging import version
 
 from moulinette import Moulinette, m18n
@@ -260,7 +261,7 @@ class BackupManager:
         backup_manager.backup()
     """
 
-    def __init__(self, name=None, description="", repositories=[], work_dir=None):
+    def __init__(self, name=None, prefix="", description="", repositories=[], work_dir=None):
         """
         BackupManager constructor
 
@@ -286,7 +287,7 @@ class BackupManager:
         self.targets = BackupRestoreTargetsManager()
 
         # Define backup name if needed
-
+        self.prefix = prefix
         if not name:
             name = self._define_backup_name()
         self.name = name
@@ -334,7 +335,7 @@ class BackupManager:
         """
         # FIXME: case where this name already exist
 
-        return time.strftime("%Y%m%d-%H%M%S", time.gmtime())
+        return self.prefix + time.strftime("%Y%m%d-%H%M%S", time.gmtime())
 
     def _init_work_dir(self):
         """Initialize preparation directory
@@ -1648,6 +1649,7 @@ class RestoreManager:
 def backup_create(
     operation_logger,
     name=None,
+    prefix="",
     description=None,
     repositories=[],
     system=[],
@@ -1679,13 +1681,15 @@ def backup_create(
 
     # Validate there is no archive with the same name
     archives = backup_list(repositories=repositories)
+    archives_already_exists = []
     for repository in archives:
         if name and name in archives[repository]:
-            repositories.pop(repository)
+            repositories.remove(repository)
+            archives_already_exists.append(repository)
             logger.error(m18n.n("backup_archive_name_exists", repository=repository))
 
     if not repositories:
-        raise YunohostValidationError("backup_archive_name_exists")
+        raise YunohostValidationError("backup_nowhere_to_backup")
 
     # If no --system or --apps given, backup everything
 
@@ -1702,7 +1706,8 @@ def backup_create(
     repositories = [BackupRepository(repo) for repo in repositories]
 
     # Prepare files to backup
-    backup_manager = BackupManager(name, description,
+    backup_manager = BackupManager(name, prefix=prefix,
+                                   description=description,
                                    repositories=repositories)
 
     # Add backup targets (system and apps)
@@ -1741,6 +1746,7 @@ def backup_create(
         )
     )
     repo_results = backup_manager.backup()
+    repo_results.update({repo: "Not sent" for repo in archives_already_exists})
     repo_states = [repo_result == "Sent" for repository, repo_result in repo_results.items()]
 
     if all(repo_states) and all(parts_states):
@@ -1978,16 +1984,19 @@ def backup_repository_remove(operation_logger, shortname, purge=False):
 
 
 @is_unit_operation()
-def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=0, keep_daily=10, keep_weekly=8, keep_monthly=8):
+def backup_repository_prune(operation_logger, shortname, prefix=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None, keep_last=None, keep_within=None):
     """
     Remove a backup repository
     """
+
     BackupRepository(shortname).prune(
         prefix=prefix,
         keep_hourly=keep_hourly,
         keep_daily=keep_daily,
         keep_weekly=keep_weekly,
         keep_monthly=keep_monthly,
+        keep_last=keep_last,
+        keep_within=keep_within,
     )
 
 
@@ -2119,7 +2128,7 @@ Group=root
     def run(self):
         self._load_current_values()
         backup_create(
-            name=self.entity,
+            prefix=f"{self.entity}_",
             description=self.description,
             repositories=self.repositories,
             system=self.system,
@@ -2128,7 +2137,7 @@ Group=root
         for repository in self.repositories:
             backup_repository_prune(
                 shortname=repository,
-                prefix=self.entity,
+                prefix=f"{self.entity}_",
                 keep_hourly=self.keep_hourly,
                 keep_daily=self.keep_daily,
                 keep_weekly=self.keep_weekly,
diff --git a/src/repositories/borg.py b/src/repositories/borg.py
index 295eeeb16..e18fdb5a4 100644
--- a/src/repositories/borg.py
+++ b/src/repositories/borg.py
@@ -176,45 +176,6 @@ class BorgBackupRepository(LocalBackupRepository):
         response = self._call('info', cmd, json_output=True)
         return response["cache"]["stats"]["unique_size"]
 
-    def prune(self, prefix=None, **kwargs):
-
-        # List archives with creation date
-        archives = {}
-        for archive_name in self.list_archive_name(prefix):
-            archive = BackupArchive(repo=self, name=archive_name)
-            created_at = archive.info()["created_at"]
-            archives[created_at] = archive
-
-        if not archives:
-            return
-
-        # Generate periods in which keep one archive
-        now = datetime.utcnow()
-        now -= timedelta(
-            minutes=now.minute,
-            seconds=now.second,
-            microseconds=now.microsecond
-        )
-        periods = set([])
-
-        for unit, qty in kwargs:
-            if not qty:
-                continue
-            period = timedelta(**{unit: 1})
-            periods += set([(now - period * i, now - period * (i - 1))
-                           for i in range(qty)])
-
-        # Delete unneeded archive
-        for created_at in sorted(archives, reverse=True):
-            created_at = datetime.utcfromtimestamp(created_at)
-            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
-
-            if keep_for:
-                periods -= keep_for
-                continue
-
-            archive.delete()
-
 
 class BorgBackupArchive(BackupArchive):
     """ Backup prepared files with borg """
diff --git a/src/repository.py b/src/repository.py
index 653043dd6..5164ff779 100644
--- a/src/repository.py
+++ b/src/repository.py
@@ -283,7 +283,30 @@ class BackupRepository(ConfigPanel):
 
         return archives
 
-    def prune(self, prefix=None, **kwargs):
+    def prune(self, prefix=None, keep_last=None, keep_within=None, keep_hourly=None, keep_daily=None, keep_weekly=None, keep_monthly=None):
+
+        # Default prune options
+        keeps = [value is None for key, value in locals().items() if key.startswith("keep_")]
+        if all(keeps):
+            keep_hourly = 0
+            keep_daily = 10
+            keep_weekly = 8
+            keep_monthly = 8
+            logger.debug(f"Prune and keep one per each {keep_hourly} last hours, {keep_daily} last days, {keep_weekly} last weeks, {keep_monthly} last months")
+
+        keep_last = keep_last if keep_last else 0
+
+        # Convert keep_within as a date
+        units = {
+            "H": "hours",
+            "d": "days",
+            "w": "weeks",
+        }
+        now = datetime.utcnow()
+        if keep_within:
+            keep_within = now - timedelta(**{units[keep_within[-1]]: int(keep_within[:-1])})
+        else:
+            keep_within = now
 
         # List archives with creation date
         archives = {}
@@ -303,24 +326,32 @@ class BackupRepository(ConfigPanel):
             microseconds=now.microsecond
         )
         periods = set([])
-
-        for unit, qty in kwargs:
+        units = {
+            "keep_hourly": {"hours": 1},
+            "keep_daily": {"days": 1},
+            "keep_weekly": {"weeks": 1},
+            "keep_monthly": {"days": 30}
+        }
+        keeps_xly = {key: val for key, val in locals().items()
+                     if key.startswith("keep_") and key.endswith("ly")}
+        for unit, qty in keeps_xly.items():
             if not qty:
                 continue
-            period = timedelta(**{unit: 1})
-            periods += set([(now - period * i, now - period * (i - 1))
-                           for i in range(qty)])
+            period = timedelta(**units[unit])
+            periods.update(set([(now - period * i, now - period * (i - 1))
+                           for i in range(qty)]))
 
         # Delete unneeded archive
         for created_at in sorted(archives, reverse=True):
-            created_at = datetime.utcfromtimestamp(created_at)
-            keep_for = set(filter(lambda period: period[0] <= created_at <= period[1], periods))
+            date_created_at = datetime.utcfromtimestamp(created_at)
+            keep_for = set(filter(lambda period: period[0] <= date_created_at <= period[1], periods))
+            periods -= keep_for
 
-            if keep_for:
-                periods -= keep_for
+            if keep_for or keep_last > 0 or date_created_at >= keep_within:
+                keep_last -= 1
                 continue
 
-            archive.delete()
+            archives[created_at].delete()
 
     # =================================================
     # Repository abstract actions