Merge branch 'stretch-unstable' into clean_helpers

data/helpers.d/getopts

@@ -148,25 +148,29 @@ ynh_handle_getopts_args () {
                                     break
                                 fi
                             else
-                                # Else, add this value to this option
+                                # Ignore empty parameters
-                                # Each value will be separated by ';'
+                                if [ -n "${all_args[$i]}" ]
-                                if [ -n "${!option_var}" ]
                                 then
-                                    # If there's already another value for this option, add a ; before adding the new value
+                                    # Else, add this value to this option
-                                    eval ${option_var}+="\;"
+                                    # Each value will be separated by ';'
+                                    if [ -n "${!option_var}" ]
+                                    then
+                                        # If there's already another value for this option, add a ; before adding the new value
+                                        eval ${option_var}+="\;"
+                                    fi
+
+                                    # Remove the \ that escape - at beginning of values.
+                                    all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}"
+
+                                    # For the record.
+                                    # We're using eval here to get the content of the variable stored itself as simple text in $option_var...
+                                    # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var}
+                                    # But... ${!option_var} can't be used as left part of an assignation.
+                                    # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself.
+                                    # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one!
+
+                                    eval ${option_var}+='"${all_args[$i]}"'
                                 fi
-
-                                # Remove the \ that escape - at beginning of values.
-                                all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}"
-
-                                # For the record.
-                                # We're using eval here to get the content of the variable stored itself as simple text in $option_var...
-                                # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var}
-                                # But... ${!option_var} can't be used as left part of an assignation.
-                                # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself.
-                                # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one!
-
-                                eval ${option_var}+='"${all_args[$i]}"'
                                 shift_value=$(( shift_value + 1 ))
                             fi
                         done

data/helpers.d/php

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-YNH_DEFAULT_PHP_VERSION=7.0
+readonly YNH_DEFAULT_PHP_VERSION=7.0
 # Declare the actual php version to use.
 # A packager willing to use another version of php can override the variable into its _common.sh.
 YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}

data/templates/nginx/server.tpl.conf

@@ -75,7 +75,7 @@ server {
     root /dev/null;
 
     location /upload/ {
-        alias /var/xmpp-upload/{{ domain }}/upload;
+        alias /var/xmpp-upload/{{ domain }}/upload/;
         # Pass all requests to metronome, except for GET and HEAD requests.
         limit_except GET HEAD {
           proxy_pass http://localhost:5290;

data/templates/postfix/main.cf

@@ -33,14 +33,20 @@ smtpd_tls_cert_file = /etc/yunohost/certs/{{ main_domain }}/crt.pem
 smtpd_tls_key_file = /etc/yunohost/certs/{{ main_domain }}/key.pem
 smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1
 smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1
-smtpd_tls_mandatory_ciphers = medium
+# smtpd_tls_mandatory_ciphers = medium # (c.f. below)
 
 # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam.pem
 # not actually 1024 bits, this applies to all DHE >= 1024 bits
 # smtpd_tls_dh1024_param_file = /path/to/dhparam.pem
 
-tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+# This custom medium cipherlist recommendation only works if we have a DH ... which we don't, c.f. https://github.com/YunoHost/issues/issues/93
+# On the other hand, the postfix doc strongly discourage tweaking this list ... So whatever, let's keep the mandatory_ciphers to high like we did before applying the Mozilla recommendation ...
+#tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
 tls_preempt_cipherlist = no
+
+# Custom Yunohost stuff ... because we can't use the recommendation about medium cipher list ...
+smtpd_tls_mandatory_ciphers=high
+smtpd_tls_eecdh_grade = ultra
 ###############################################################################
 smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
 smtpd_tls_loglevel=1

src/yunohost/app.py

@@ -512,7 +512,7 @@ def app_upgrade(app=[], url=None, file=None):
             upgrade_failed = True if upgrade_retcode != 0 else False
             if upgrade_failed:
                 error = m18n.n('app_upgrade_script_failed')
-                logger.exception(m18n.n("app_upgrade_failed", app=app_instance_name, error=error))
+                logger.error(m18n.n("app_upgrade_failed", app=app_instance_name, error=error))
                 failure_message_with_debug_instructions = operation_logger.error(error)
                 if msettings.get('interface') != 'api':
                     dump_app_log_extract_for_debugging(operation_logger)
@@ -520,13 +520,13 @@ def app_upgrade(app=[], url=None, file=None):
         except (KeyboardInterrupt, EOFError):
             upgrade_retcode = -1
             error = m18n.n('operation_interrupted')
-            logger.exception(m18n.n("app_upgrade_failed", app=app_instance_name, error=error))
+            logger.error(m18n.n("app_upgrade_failed", app=app_instance_name, error=error))
             failure_message_with_debug_instructions = operation_logger.error(error)
         # Something wrong happened in Yunohost's code (most probably hook_exec)
         except Exception:
             import traceback
             error = m18n.n('unexpected_error', error=u"\n" + traceback.format_exc())
-            logger.exception(m18n.n("app_install_failed", app=app_instance_name, error=error))
+            logger.error(m18n.n("app_install_failed", app=app_instance_name, error=error))
             failure_message_with_debug_instructions = operation_logger.error(error)
         finally:
             # Whatever happened (install success or failure) we check if it broke the system
@@ -536,7 +536,7 @@ def app_upgrade(app=[], url=None, file=None):
                 _assert_system_is_sane_for_app(manifest, "post")
             except Exception as e:
                 broke_the_system = True
-                logger.exception(m18n.n("app_upgrade_failed", app=app_instance_name, error=str(e)))
+                logger.error(m18n.n("app_upgrade_failed", app=app_instance_name, error=str(e)))
                 failure_message_with_debug_instructions = operation_logger.error(str(e))
 
             # If upgrade failed or broke the system,
@@ -768,20 +768,20 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
         install_failed = True if install_retcode != 0 else False
         if install_failed:
             error = m18n.n('app_install_script_failed')
-            logger.exception(m18n.n("app_install_failed", app=app_id, error=error))
+            logger.error(m18n.n("app_install_failed", app=app_id, error=error))
             failure_message_with_debug_instructions = operation_logger.error(error)
             if msettings.get('interface') != 'api':
                 dump_app_log_extract_for_debugging(operation_logger)
     # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception
     except (KeyboardInterrupt, EOFError):
         error = m18n.n('operation_interrupted')
-        logger.exception(m18n.n("app_install_failed", app=app_id, error=error))
+        logger.error(m18n.n("app_install_failed", app=app_id, error=error))
         failure_message_with_debug_instructions = operation_logger.error(error)
     # Something wrong happened in Yunohost's code (most probably hook_exec)
     except Exception as e:
         import traceback
         error = m18n.n('unexpected_error', error=u"\n" + traceback.format_exc())
-        logger.exception(m18n.n("app_install_failed", app=app_id, error=error))
+        logger.error(m18n.n("app_install_failed", app=app_id, error=error))
         failure_message_with_debug_instructions = operation_logger.error(error)
     finally:
         # Whatever happened (install success or failure) we check if it broke the system
@@ -791,7 +791,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
             _assert_system_is_sane_for_app(manifest, "post")
         except Exception as e:
             broke_the_system = True
-            logger.exception(m18n.n("app_install_failed", app=app_id, error=str(e)))
+            logger.error(m18n.n("app_install_failed", app=app_id, error=str(e)))
             failure_message_with_debug_instructions = operation_logger.error(str(e))
 
         # If the install failed or broke the system, we remove it
@@ -828,7 +828,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
             except (KeyboardInterrupt, EOFError, Exception):
                 remove_retcode = -1
                 import traceback
-                logger.exception(m18n.n('unexpected_error', error=u"\n" + traceback.format_exc()))
+                logger.error(m18n.n('unexpected_error', error=u"\n" + traceback.format_exc()))
 
             # Remove all permission in LDAP
             for permission_name in user_permission_list()["permissions"].keys():
@@ -999,7 +999,7 @@ def app_remove(operation_logger, app):
     except (KeyboardInterrupt, EOFError, Exception):
         ret = -1
         import traceback
-        logger.exception(m18n.n('unexpected_error', error=u"\n" + traceback.format_exc()))
+        logger.error(m18n.n('unexpected_error', error=u"\n" + traceback.format_exc()))
 
     if ret == 0:
         logger.success(m18n.n('app_removed', app=app))
@@ -1825,7 +1825,7 @@ def _get_app_settings(app_id):
         if app_id == settings['id']:
             return settings
     except (IOError, TypeError, KeyError):
-        logger.exception(m18n.n('app_not_correctly_installed',
+        logger.error(m18n.n('app_not_correctly_installed',
                                 app=app_id))
     return {}
 

src/yunohost/backup.py

@@ -35,9 +35,9 @@ import tempfile
 from datetime import datetime
 from glob import glob
 from collections import OrderedDict
+from functools import reduce
 
 from moulinette import msignals, m18n, msettings
-from yunohost.utils.error import YunohostError
 from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml
@@ -51,7 +51,8 @@ from yunohost.hook import (
 from yunohost.tools import tools_postinstall
 from yunohost.regenconf import regen_conf
 from yunohost.log import OperationLogger
-from functools import reduce
+from yunohost.utils.error import YunohostError
+from yunohost.utils.packages import ynh_packages_version
 
 BACKUP_PATH = '/home/yunohost.backup'
 ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
@@ -282,7 +283,8 @@ class BackupManager():
             'size': self.size,
             'size_details': self.size_details,
             'apps': self.apps_return,
-            'system': self.system_return
+            'system': self.system_return,
+            'from_yunohost_version': ynh_packages_version()["yunohost"]["version"]
         }
 
     @property
@@ -604,7 +606,7 @@ class BackupManager():
         ret_succeed = {hook: [path for path, result in infos.items() if result["state"] == "succeed"]
                        for hook, infos in ret.items()
                        if any(result["state"] == "succeed" for result in infos.values())}
-        ret_failed = {hook: [path for path, result in infos.items.items() if result["state"] == "failed"]
+        ret_failed = {hook: [path for path, result in infos.items() if result["state"] == "failed"]
                       for hook, infos in ret.items()
                       if any(result["state"] == "failed" for result in infos.values())}
 

src/yunohost/utils/yunopaste.py

@@ -37,10 +37,18 @@ def yunopaste(data):
 
 def anonymize(data):
 
+    def anonymize_domain(data, domain, redact):
+        data = data.replace(domain, redact)
+        # This stuff appears sometimes because some folder in
+        # /var/lib/metronome/ have some folders named this way
+        data = data.replace(domain.replace(".", "%2e"), redact.replace(".", "%2e"))
+        return data
+
+
     # First, let's replace every occurence of the main domain by "domain.tld"
     # This should cover a good fraction of the info leaked
     main_domain = _get_maindomain()
-    data = data.replace(main_domain, "maindomain.tld")
+    data = anonymize_domain(data, main_domain, "maindomain.tld")
 
     # Next, let's replace other domains. We do this in increasing lengths,
     # because e.g. knowing that the domain is a sub-domain of another domain may
@@ -55,7 +63,7 @@ def anonymize(data):
     for domain in domains:
         if domain not in data:
             continue
-        data = data.replace(domain, "domain%s.tld" % count)
+        data = anonymize_domain(data, domain, "domain%s.tld" % count)
         count += 1
 
     # We also want to anonymize the ips