Merge branch 'unstable' into regen-conf

data/actionsmap/yunohost.yml

@@ -596,6 +596,9 @@ backup:
                 --hooks:
                     help: List of backup hooks names to execute
                     nargs: "*"
+                --apps:
+                    help: List of application names to backup
+                    nargs: "*"
                 --ignore-apps:
                     help: Do not backup apps
                     action: store_true
@@ -603,15 +606,16 @@ backup:
         ### backup_restore()
         restore:
             action_help: Restore from a local backup archive
-            api: POST /restore
-            configuration:
-                authenticate: false
+            api: POST /backup/restore/<name>
             arguments:
                 name:
                     help: Name of the local backup archive
                 --hooks:
                     help: List of restauration hooks names to execute
                     nargs: "*"
+                --apps:
+                    help: List of application names to restore
+                    nargs: "*"
                 --ignore-apps:
                     help: Do not restore apps
                     action: store_true
@@ -625,6 +629,15 @@ backup:
             api: GET /backup/archives
             configuration:
                 lock: false
+            arguments:
+                -i:
+                    full: --with-info
+                    help: Show backup information for each archive
+                    action: store_true
+                -H:
+                    full: --human-readable
+                    help: Print sizes in human readable format
+                    action: store_true
 
         ### backup_info()
         info:
@@ -635,6 +648,14 @@ backup:
             arguments:
                 name:
                     help: Name of the local backup archive
+                -d:
+                    full: --with-details
+                    help: Show additional backup information
+                    action: store_true
+                -H:
+                    full: --human-readable
+                    help: Print sizes in human readable format
+                    action: store_true
 
 
 #############################

data/hooks/restore/11-system_mysql

@@ -1,5 +1,5 @@
 backup_dir="$1/mysql"
 
 mysqlpwd=$(sudo cat /etc/yunohost/mysql)
-sudo mysql -uroot -p"$mysqlpwd" mysql < $backup_dir/mysql.sql
+#sudo mysql -uroot -p"$mysqlpwd" mysql < $backup_dir/mysql.sql
 sudo mysqladmin flush-privileges -p"$mysqlpwd"

debian/control

@@ -9,17 +9,22 @@ Homepage: https://yunohost.org/
 Package: moulinette-yunohost
 Architecture: all
 Depends: moulinette (>= 2.2.1),
-    python-psutil,
-    python-requests,
-    glances,
-    python-pip,
-    pyminiupnpc,
-    dnsutils,
-    bind9utils,
-    python-apt,
-    ca-certificates,
-    python-dnspython,
-    netcat-openbsd,
-    iproute
+ python-psutil,
+ python-requests,
+ glances,
+ python-pip,
+ pyminiupnpc,
+ dnsutils,
+ bind9utils,
+ python-apt,
+ ca-certificates,
+ python-dnspython,
+ netcat-openbsd,
+ iproute,
+ unzip,
+ git-core,
+ curl,
+ mariadb-server | mysql-server, php5-mysql | php5-mysqlnd
+Conflicts: iptables-persistent
 Description: YunoHost Python scripts
  Python functions to manage a YunoHost instance

lib/yunohost/app.py

@@ -351,7 +351,8 @@ def app_upgrade(auth, app=[], url=None, file=None):
                             for line in lines:
                                 sources.write(re.sub(r''+ original_app_id +'', app_id, line))
 
-        # Add hooks
+        # Clean hooks and add new ones
+        hook_remove(app_id)
         if 'hooks' in os.listdir(app_tmp_folder):
             for hook in os.listdir(app_tmp_folder +'/hooks'):
                 hook_add(app_id, app_tmp_folder +'/hooks/'+ hook)
@@ -451,7 +452,8 @@ def app_install(auth, app, label=None, args=None):
     os.makedirs(app_setting_path)
     os.system('touch %s/settings.yml' % app_setting_path)
 
-    # Add hooks
+    # Clean hooks and add new ones
+    hook_remove(app_id)
     if 'hooks' in os.listdir(app_tmp_folder):
         for file in os.listdir(app_tmp_folder +'/hooks'):
             hook_add(app_id, app_tmp_folder +'/hooks/'+ file)
@@ -1034,7 +1036,7 @@ def _fetch_app_from_git(app):
             if os.system('wget "%s" -O "%s.zip" > /dev/null 2>&1' % (url, app_tmp_folder)) == 0:
                 return _extract_app_from_file(app_tmp_folder +'.zip', remove=True)
 
-        git_result   = os.system('git clone %s %s' % (app, app_tmp_folder))
+        git_result   = os.system('git clone --depth=1 %s %s' % (app, app_tmp_folder))
         git_result_2 = 0
         try:
             with open(app_tmp_folder + '/manifest.json') as json_manifest:

lib/yunohost/backup.py

@@ -29,8 +29,9 @@ import sys
 import json
 import errno
 import time
-import shutil
 import tarfile
+import subprocess
+from collections import OrderedDict
 
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
@@ -42,7 +43,7 @@ logger = getActionLogger('yunohost.backup')
 
 
 def backup_create(name=None, description=None, output_directory=None,
-                  no_compress=False, hooks=[], ignore_apps=False):
+                  no_compress=False, hooks=[], apps=[], ignore_apps=False):
     """
     Create a backup local archive
 
@@ -52,12 +53,12 @@ def backup_create(name=None, description=None, output_directory=None,
         output_directory -- Output directory for the backup
         no_compress -- Do not create an archive file
         hooks -- List of backup hooks names to execute
+        apps -- List of application names to backup
         ignore_apps -- Do not backup apps
 
     """
     # TODO: Add a 'clean' argument to clean output directory
-    from yunohost.hook import hook_add
-    from yunohost.hook import hook_callback
+    from yunohost.hook import hook_callback, hook_exec
 
     tmp_dir = None
 
@@ -121,33 +122,58 @@ def backup_create(name=None, description=None, output_directory=None,
         'description': description or '',
         'created_at': timestamp,
         'apps': {},
+        'hooks': {},
     }
 
-    # Add apps backup hook
+    # Run system hooks
+    msignals.display(m18n.n('backup_running_hooks'))
+    hooks_ret = hook_callback('backup', hooks, args=[tmp_dir])
+
+    # Add hooks results to the info
+    info['hooks'] = hooks_ret['succeed']
+
+    # Backup apps
     if not ignore_apps:
         from yunohost.app import app_info
-        try:
-            for app_id in os.listdir('/etc/yunohost/apps'):
-                hook = '/etc/yunohost/apps/%s/scripts/backup' % app_id
-                if os.path.isfile(hook):
-                    hook_add(app_id, hook)
 
-                    # Add app info
-                    i = app_info(app_id)
-                    info['apps'][app_id] = {
-                        'version': i['version'],
-                    }
+        # Filter applications to backup
+        apps_list = set(os.listdir('/etc/yunohost/apps'))
+        apps_filtered = set()
+        if apps:
+            for a in apps:
+                if a not in apps_list:
+                    logger.warning("app '%s' not found", a)
+                    msignals.display(m18n.n('unbackup_app', a), 'warning')
                 else:
-                    logger.warning("unable to find app's backup hook '%s'",
-                                   hook)
-                    msignals.display(m18n.n('unbackup_app', app_id),
-                                     'warning')
-        except IOError as e:
-            logger.info("unable to add apps backup hook: %s", str(e))
+                    apps_filtered.add(a)
+        else:
+            apps_filtered = apps_list
 
-    # Run hooks
-    msignals.display(m18n.n('backup_running_hooks'))
-    hook_callback('backup', hooks, args=[tmp_dir])
+        # Run apps backup scripts
+        tmp_script = '/tmp/backup_' + str(timestamp)
+        for app_id in apps_filtered:
+            script = '/etc/yunohost/apps/{:s}/scripts/backup'.format(app_id)
+            if not os.path.isfile(script):
+                logger.warning("backup script '%s' not found", script)
+                msignals.display(m18n.n('unbackup_app', app_id),
+                                 'warning')
+                continue
+
+            try:
+                msignals.display(m18n.n('backup_running_app_script', app_id))
+                subprocess.call(['install', '-Dm555', script, tmp_script])
+                hook_exec(tmp_script, args=[tmp_dir])
+            except:
+                logger.exception("error while executing script '%s'", script)
+                msignals.display(m18n.n('unbackup_app', app_id),
+                                 'error')
+            else:
+                # Add app info
+                i = app_info(app_id)
+                info['apps'][app_id] = {
+                    'version': i['version'],
+                }
+        subprocess.call(['rm', '-f', tmp_script])
 
     # Create backup info file
     with open("%s/info.json" % tmp_dir, 'w') as f:
@@ -191,14 +217,19 @@ def backup_create(name=None, description=None, output_directory=None,
 
     msignals.display(m18n.n('backup_complete'), 'success')
 
+    # Return backup info
+    info['name'] = name
+    return { 'archive': info }
 
-def backup_restore(name, hooks=[], ignore_apps=False, force=False):
+
+def backup_restore(name, hooks=[], apps=[], ignore_apps=False, force=False):
     """
     Restore from a local backup archive
 
     Keyword argument:
         name -- Name of the local backup archive
         hooks -- List of restoration hooks names to execute
+        apps -- List of application names to restore
         ignore_apps -- Do not restore apps
         force -- Force restauration on an already installed system
 
@@ -239,15 +270,6 @@ def backup_restore(name, hooks=[], ignore_apps=False, force=False):
         logger.info("restoring from backup '%s' created on %s", name,
                     time.ctime(info['created_at']))
 
-    # Retrieve domain from the backup
-    try:
-        with open("%s/yunohost/current_host" % tmp_dir, 'r') as f:
-            domain = f.readline().rstrip()
-    except IOError:
-        logger.error("unable to retrieve domain from '%s/yunohost/current_host'",
-                     tmp_dir)
-        raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
-
     # Check if YunoHost is installed
     if os.path.isfile('/etc/yunohost/installed'):
         msignals.display(m18n.n('yunohost_already_installed'), 'warning')
@@ -265,12 +287,35 @@ def backup_restore(name, hooks=[], ignore_apps=False, force=False):
                 raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed'))
     else:
         from yunohost.tools import tools_postinstall
+
+        # Retrieve the domain from the backup
+        try:
+            with open("%s/yunohost/current_host" % tmp_dir, 'r') as f:
+                domain = f.readline().rstrip()
+        except IOError:
+            logger.error("unable to retrieve domain from '%s/yunohost/current_host'",
+                         tmp_dir)
+            raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
+
         logger.info("executing the post-install...")
         tools_postinstall(domain, 'yunohost', True)
 
     # Add apps restore hook
     if not ignore_apps:
-        for app_id in info['apps'].keys():
+        # Filter applications to restore
+        apps_list = set(info['apps'].keys())
+        apps_filtered = set()
+        if apps:
+            for a in apps:
+                if a not in apps_list:
+                    logger.warning("app '%s' not found", a)
+                    msignals.display(m18n.n('unrestore_app', a), 'warning')
+                else:
+                    apps_filtered.add(a)
+        else:
+            apps_filtered = apps_list
+
+        for app_id in apps_filtered:
             hook = "/etc/yunohost/apps/%s/scripts/restore" % app_id
             if os.path.isfile(hook):
                 hook_add(app_id, hook)
@@ -288,10 +333,14 @@ def backup_restore(name, hooks=[], ignore_apps=False, force=False):
     msignals.display(m18n.n('restore_complete'), 'success')
 
 
-def backup_list():
+def backup_list(with_info=False, human_readable=False):
     """
     List available local backup archives
 
+    Keyword arguments:
+        with_info -- Show backup information for each archive
+        human_readable -- Print sizes in human readable format
+
     """
     result = []
 
@@ -308,18 +357,29 @@ def backup_list():
             except ValueError:
                 continue
             result.append(name)
+        result.sort()
+
+    if result and with_info:
+        d = OrderedDict()
+        for a in result:
+            d[a] = backup_info(a, human_readable=human_readable)
+        result = d
 
     return { 'archives': result }
 
 
-def backup_info(name):
+def backup_info(name, with_details=False, human_readable=False):
     """
     Get info about a local backup archive
 
     Keyword arguments:
         name -- Name of the local backup archive
+        with_details -- Show additional backup information
+        human_readable -- Print sizes in human readable format
 
     """
+    from yunohost.monitor import binary_to_human
+
     archive_file = '%s/%s.tar.gz' % (archives_path, name)
     if not os.path.isfile(archive_file):
         logger.error("no local backup archive found at '%s'", archive_file)
@@ -336,10 +396,19 @@ def backup_info(name):
                          info_file)
         raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
 
-    return {
+    size = os.path.getsize(archive_file)
+    if human_readable:
+        size = binary_to_human(size) + 'B'
+
+    result = {
         'path': archive_file,
         'created_at': time.strftime(m18n.n('format_datetime_short'),
                                     time.gmtime(info['created_at'])),
         'description': info['description'],
-        'apps': info['apps'],
+        'size': size,
     }
+
+    if with_details:
+        for d in ['apps', 'hooks']:
+            result[d] = info[d]
+    return result

lib/yunohost/hook.py

@@ -174,7 +174,7 @@ def hook_callback(action, hooks=[], args=None):
         args -- Ordered list of arguments to pass to the script
 
     """
-    result = { 'succeed': list(), 'failed': list() }
+    result = { 'succeed': {}, 'failed': {} }
     hooks_dict = {}
 
     # Retrieve hooks
@@ -220,15 +220,18 @@ def hook_callback(action, hooks=[], args=None):
     # Iterate over hooks and execute them
     for priority in sorted(hooks_dict):
         for name, info in iter(hooks_dict[priority].items()):
+            state = 'succeed'
             filename = '%s-%s' % (priority, name)
             try:
                 hook_exec(info['path'], args=args)
             except:
                 logger.exception("error while executing hook '%s'",
                                  info['path'])
-                result['failed'].append(filename)
-            else:
-                result['succeed'].append(filename)
+                state = 'failed'
+            try:
+                result[state][name].append(info['path'])
+            except KeyError:
+                result[state][name] = [info['path']]
     return result
 
 

lib/yunohost/monitor.py

@@ -130,7 +130,7 @@ def monitor_disk(units=None, mountpoint=None, human_readable=False):
                 else:
                     if human_readable:
                         for i in ['used', 'avail', 'size']:
-                            d[i] = _binary_to_human(d[i]) + 'B'
+                            d[i] = binary_to_human(d[i]) + 'B'
                     _set(dname, d)
             for dname in devices_names:
                 _set(dname, 'not-available')
@@ -201,7 +201,7 @@ def monitor_network(units=None, human_readable=False):
                     if human_readable:
                         for k in i.keys():
                             if k != 'time_since_update':
-                                i[k] = _binary_to_human(i[k]) + 'B'
+                                i[k] = binary_to_human(i[k]) + 'B'
                     result[u][iname] = i
         elif u == 'infos':
             try:
@@ -261,10 +261,10 @@ def monitor_system(units=None, human_readable=False):
             if human_readable:
                 for i in ram.keys():
                     if i != 'percent':
-                        ram[i] = _binary_to_human(ram[i]) + 'B'
+                        ram[i] = binary_to_human(ram[i]) + 'B'
                 for i in swap.keys():
                     if i != 'percent':
-                        swap[i] = _binary_to_human(swap[i]) + 'B'
+                        swap[i] = binary_to_human(swap[i]) + 'B'
             result[u] = {
                 'ram': ram,
                 'swap': swap
@@ -510,7 +510,7 @@ def _extract_inet(string, skip_netmask=False, skip_loopback=True):
     return result
 
 
-def _binary_to_human(n, customary=False):
+def binary_to_human(n, customary=False):
     """
     Convert bytes or bits into human readable format with binary prefix
 

locales/en.json

@@ -143,6 +143,7 @@
     "backup_output_directory_forbidden" : "Forbidden output directory",
     "backup_output_directory_not_empty" : "Output directory is not empty",
     "backup_running_hooks" : "Running backup hooks...",
+    "backup_running_app_script" : "Running backup script of app '{:s}'...",
     "backup_creating_archive" : "Creating the backup archive...",
     "backup_extracting_archive" : "Extracting the backup archive...",
     "backup_archive_open_failed" : "Unable to open the backup archive",