Merge branch 'stretch-unstable' into featureCipherStrategy

bin/yunohost-api

@@ -149,6 +149,11 @@ def _init_moulinette(use_websocket=True, debug=False, verbose=False):
                 'handlers': [],
                 'propagate': True,
             },
+            'gnupg': {
+                'level': 'INFO',
+                'handlers': [],
+                'propagate': False,
+            },
         },
         'root': {
             'level': level,
@@ -196,12 +201,10 @@ if __name__ == '__main__':
     _init_moulinette(opts.use_websocket, opts.debug, opts.verbose)
 
     # Run the server
-    from yunohost.utils.packages import ynh_packages_version
     ret = moulinette.api(
         _retrieve_namespaces(),
         host=opts.host, port=opts.port, routes={
             ('GET', '/installed'): is_installed,
-            ('GET', '/version'): ynh_packages_version,
         }, use_cache=opts.use_cache, use_websocket=opts.use_websocket
     )
     sys.exit(ret)

data/helpers.d/backend

@@ -1,9 +1,11 @@
+#!/bin/bash
+
 # Use logrotate to manage the logfile
 #
-# usage: ynh_use_logrotate [logfile] [--non-append|--append] [specific_user/specific_group]
+# usage: ynh_use_logrotate [--logfile=/log/file] [--nonappend] [--specific_user=user/group]
-# | arg: logfile - absolute path of logfile
+# | arg: -l, --logfile= - absolute path of logfile
-# | arg: --non-append - (Option) Replace the config file instead of appending this new config.
+# | arg: -n, --nonappend - (Option) Replace the config file instead of appending this new config.
-# | arg: specific_user : run logrotate as the specified user and group. If not specified logrotate is runned as root.
+# | arg: -u, --specific_user : run logrotate as the specified user and group. If not specified logrotate is runned as root.
 #
 # If no argument provided, a standard directory will be use. /var/log/${app}
 # You can provide a path with the directory only or with the logfile.
@@ -13,28 +15,53 @@
 # It's possible to use this helper several times, each config will be added to the same logrotate config file.
 # Unless you use the option --non-append
 ynh_use_logrotate () {
-	local customtee="tee -a"
+	# Declare an array to define the options of this helper.
-	local user_group="${3:-}"
+	local legacy_args=lnuya
+	declare -Ar args_array=( [l]=logfile= [n]=nonappend [u]=specific_user= [y]=non [a]=append )
+	# [y]=non [a]=append are only for legacy purpose, to not fail on the old option '--non-append'
+	local logfile
+	local nonappend
+	local specific_user
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	local logfile="${logfile:-}"
+	local nonappend="${nonappend:-0}"
+	local specific_user="${specific_user:-}"
+
+	# LEGACY CODE - PRE GETOPTS
 	if [ $# -gt 0 ] && [ "$1" == "--non-append" ]; then
-		customtee="tee"
+		nonappend=1
 		# Destroy this argument for the next command.
 		shift
 	elif [ $# -gt 1 ] && [ "$2" == "--non-append" ]; then
-		customtee="tee"
+		nonappend=1
 	fi
-	if [ $# -gt 0 ]; then
+
+	if [ $# -gt 0 ] && [ "$(echo ${1:0:1})" != "-" ]; then
 		if [ "$(echo ${1##*.})" == "log" ]; then	# Keep only the extension to check if it's a logfile
 			local logfile=$1	# In this case, focus logrotate on the logfile
 		else
 			local logfile=$1/*.log	# Else, uses the directory and all logfile into it.
 		fi
+	fi
+	# LEGACY CODE
+
+	local customtee="tee -a"
+	if [ "$nonappend" -eq 1 ]; then
+		customtee="tee"
+	fi
+	if [ -n "$logfile" ]
+	then
+		if [ "$(echo ${logfile##*.})" != "log" ]; then	# Keep only the extension to check if it's a logfile
+			local logfile="$1/*.log"	# Else, uses the directory and all logfile into it.
+		fi
 	else
-		local logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log
+		logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log
 	fi
 	local su_directive=""
-	if [[ -n $user_group ]]; then
+	if [[ -n $specific_user ]]; then
 		su_directive="	# Run logorotate as specific user - group
-	su ${user_group%/*} ${user_group#*/}"
+	su ${specific_user%/*} ${specific_user#*/}"
 	fi
 
 	cat > ./${app}-logrotate << EOF	# Build a config file for logrotate
@@ -73,9 +100,9 @@ ynh_remove_logrotate () {
 
 # Create a dedicated systemd config
 #
-# usage: ynh_add_systemd_config [service] [template]
+# usage: ynh_add_systemd_config [--service=service] [--template=template]
-# | arg: service - Service name (optionnal, $app by default)
+# | arg: -s, --service - Service name (optionnal, $app by default)
-# | arg: template - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template)
+# | arg: -t, --template - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template)
 #
 # This will use the template ../conf/<templatename>.service
 # to generate a systemd config, by replacing the following keywords
@@ -86,40 +113,54 @@ ynh_remove_logrotate () {
 #   __FINALPATH__ by  $final_path
 #
 ynh_add_systemd_config () {
-	local service_name="${1:-$app}"
+	# Declare an array to define the options of this helper.
+	local legacy_args=st
+	declare -Ar args_array=( [s]=service= [t]=template= )
+	local service
+	local template
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	local service="${service:-$app}"
+	local template="${template:-systemd.service}"
 
-	finalsystemdconf="/etc/systemd/system/$service_name.service"
+	finalsystemdconf="/etc/systemd/system/$service.service"
-	ynh_backup_if_checksum_is_different "$finalsystemdconf"
+	ynh_backup_if_checksum_is_different --file="$finalsystemdconf"
-	sudo cp ../conf/${2:-systemd.service} "$finalsystemdconf"
+	sudo cp ../conf/$template "$finalsystemdconf"
 
 	# To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
 	# Substitute in a nginx config file only if the variable is not empty
 	if test -n "${final_path:-}"; then
-		ynh_replace_string "__FINALPATH__" "$final_path" "$finalsystemdconf"
+		ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalsystemdconf"
 	fi
 	if test -n "${app:-}"; then
-		ynh_replace_string "__APP__" "$app" "$finalsystemdconf"
+		ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finalsystemdconf"
 	fi
-	ynh_store_file_checksum "$finalsystemdconf"
+	ynh_store_file_checksum --file="$finalsystemdconf"
 
 	sudo chown root: "$finalsystemdconf"
-	sudo systemctl enable $service_name
+	sudo systemctl enable $service
 	sudo systemctl daemon-reload
 }
 
 # Remove the dedicated systemd config
 #
-# usage: ynh_remove_systemd_config [service]
+# usage: ynh_remove_systemd_config [--service=service]
-# | arg: service - Service name (optionnal, $app by default)
+# | arg: -s, --service - Service name (optionnal, $app by default)
 #
 ynh_remove_systemd_config () {
-	local service_name="${1:-$app}"
+	# Declare an array to define the options of this helper.
+	local legacy_args=s
+	declare -Ar args_array=( [s]=service= )
+	local service
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	local service="${service:-$app}"
 
-	local finalsystemdconf="/etc/systemd/system/$service_name.service"
+	local finalsystemdconf="/etc/systemd/system/$service.service"
 	if [ -e "$finalsystemdconf" ]; then
-		sudo systemctl stop $service_name
+		sudo systemctl stop $service
-		sudo systemctl disable $service_name
+		sudo systemctl disable $service
-		ynh_secure_remove "$finalsystemdconf"
+		ynh_secure_remove --file="$finalsystemdconf"
 		sudo systemctl daemon-reload
 	fi
 }
@@ -145,7 +186,7 @@ ynh_remove_systemd_config () {
 ynh_add_nginx_config () {
 	finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf"
 	local others_var=${1:-}
-	ynh_backup_if_checksum_is_different "$finalnginxconf"
+	ynh_backup_if_checksum_is_different --file="$finalnginxconf"
 	sudo cp ../conf/nginx.conf "$finalnginxconf"
 
 	# To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
@@ -153,20 +194,20 @@ ynh_add_nginx_config () {
 	if test -n "${path_url:-}"; then
 		# path_url_slash_less is path_url, or a blank value if path_url is only '/'
 		local path_url_slash_less=${path_url%/}
-		ynh_replace_string "__PATH__/" "$path_url_slash_less/" "$finalnginxconf"
+		ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$finalnginxconf"
-		ynh_replace_string "__PATH__" "$path_url" "$finalnginxconf"
+		ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finalnginxconf"
 	fi
 	if test -n "${domain:-}"; then
-		ynh_replace_string "__DOMAIN__" "$domain" "$finalnginxconf"
+		ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$finalnginxconf"
 	fi
 	if test -n "${port:-}"; then
-		ynh_replace_string "__PORT__" "$port" "$finalnginxconf"
+		ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$finalnginxconf"
 	fi
 	if test -n "${app:-}"; then
-		ynh_replace_string "__NAME__" "$app" "$finalnginxconf"
+		ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$finalnginxconf"
 	fi
 	if test -n "${final_path:-}"; then
-		ynh_replace_string "__FINALPATH__" "$final_path" "$finalnginxconf"
+		ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalnginxconf"
 	fi
 
 	# Replace all other variable given as arguments
@@ -174,17 +215,17 @@ ynh_add_nginx_config () {
 	do
 		# ${var_to_replace^^} make the content of the variable on upper-cases
 		# ${!var_to_replace} get the content of the variable named $var_to_replace 
-		ynh_replace_string "__${var_to_replace^^}__" "${!var_to_replace}" "$finalnginxconf"
+		ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalnginxconf"
 	done
 	
 	if [ "${path_url:-}" != "/" ]
 	then
-		ynh_replace_string "^#sub_path_only" "" "$finalnginxconf"
+		ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf"
 	else
-		ynh_replace_string "^#root_path_only" "" "$finalnginxconf"
+		ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf"
 	fi
 
-	ynh_store_file_checksum "$finalnginxconf"
+	ynh_store_file_checksum --file="$finalnginxconf"
 
 	sudo systemctl reload nginx
 }
@@ -193,7 +234,7 @@ ynh_add_nginx_config () {
 #
 # usage: ynh_remove_nginx_config
 ynh_remove_nginx_config () {
-	ynh_secure_remove "/etc/nginx/conf.d/$domain.d/$app.conf"
+	ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf"
 	sudo systemctl reload nginx
 }
 
@@ -209,16 +250,16 @@ ynh_add_fpm_config () {
 		fpm_config_dir="/etc/php5/fpm"
 		fpm_service="php5-fpm"
 	fi
-	ynh_app_setting_set $app fpm_config_dir "$fpm_config_dir"
+	ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir"
-	ynh_app_setting_set $app fpm_service "$fpm_service"
+	ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service"
 	finalphpconf="$fpm_config_dir/pool.d/$app.conf"
-	ynh_backup_if_checksum_is_different "$finalphpconf"
+	ynh_backup_if_checksum_is_different --file="$finalphpconf"
 	sudo cp ../conf/php-fpm.conf "$finalphpconf"
-	ynh_replace_string "__NAMETOCHANGE__" "$app" "$finalphpconf"
+	ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf"
-	ynh_replace_string "__FINALPATH__" "$final_path" "$finalphpconf"
+	ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf"
-	ynh_replace_string "__USER__" "$app" "$finalphpconf"
+	ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf"
 	sudo chown root: "$finalphpconf"
-	ynh_store_file_checksum "$finalphpconf"
+	ynh_store_file_checksum --file="$finalphpconf"
 
 	if [ -e "../conf/php-fpm.ini" ]
 	then
@@ -236,14 +277,160 @@ ynh_add_fpm_config () {
 #
 # usage: ynh_remove_fpm_config
 ynh_remove_fpm_config () {
-	local fpm_config_dir=$(ynh_app_setting_get $app fpm_config_dir)
+	local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
-	local fpm_service=$(ynh_app_setting_get $app fpm_service)
+	local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service)
 	# Assume php version 7 if not set
 	if [ -z "$fpm_config_dir" ]; then
 		fpm_config_dir="/etc/php/7.0/fpm"
 		fpm_service="php7.0-fpm"
 	fi
-	ynh_secure_remove "$fpm_config_dir/pool.d/$app.conf"
+	ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf"
-	ynh_secure_remove "$fpm_config_dir/conf.d/20-$app.ini" 2>&1
+	ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1
 	sudo systemctl reload $fpm_service
 }
+
+# Create a dedicated fail2ban config (jail and filter conf files)
+#
+# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports]
+# | arg: -l, --logpath=   - Log file to be checked by fail2ban
+# | arg: -r, --failregex= - Failregex to be looked for by fail2ban
+# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3
+# | arg: -p, --ports=     - Ports blocked for a banned IP address - default: http,https
+#
+# -----------------------------------------------------------------------------
+#
+# usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"]
+# | arg: -t, --use_template - Use this helper in template mode
+# | arg: -v, --others_var=  - List of others variables to replace separeted by a space
+# |                           for example : 'var_1 var_2 ...'
+#
+# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf
+#   __APP__      by  $app
+#
+#  You can dynamically replace others variables by example :
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+#
+# Generally your template will look like that by example (for synapse):
+#
+# f2b_jail.conf:
+#     [__APP__]
+#     enabled = true
+#     port = http,https
+#     filter = __APP__
+#     logpath = /var/log/__APP__/logfile.log
+#     maxretry = 3
+#
+# f2b_filter.conf:
+#     [INCLUDES]
+#     before = common.conf
+#     [Definition]
+#
+#     # Part of regex definition (just used to make more easy to make the global regex)
+#     __synapse_start_line = .? \- synapse\..+ \-
+#
+#    # Regex definition.
+#    failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- <HOST> \- \d+ \- Received request\: POST /_matrix/client/r0/login\??<SKIPLINES>%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'<SKIPLINES>%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$
+#
+#     ignoreregex =
+#
+# -----------------------------------------------------------------------------
+#
+# Note about the "failregex" option:
+#          regex to match the password failure messages in the logfile. The
+#          host must be matched by a group named "host". The tag "<HOST>" can
+#          be used for standard IP/hostname matching and is only an alias for
+#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
+#
+#          You can find some more explainations about how to make a regex here :
+#          https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters
+#
+# Note that the logfile need to exist before to call this helper !!
+#
+# To validate your regex you can test with this command:
+# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf
+#
+ynh_add_fail2ban_config () {
+  # Declare an array to define the options of this helper.
+  declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=)
+  local logpath
+  local failregex
+  local max_retry
+  local ports
+  local others_var
+  local use_template
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
+  use_template="${use_template:-0}"
+  max_retry=${max_retry:-3}
+  ports=${ports:-http,https}
+
+  finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf"
+  finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf"
+  ynh_backup_if_checksum_is_different "$finalfail2banjailconf"
+  ynh_backup_if_checksum_is_different "$finalfail2banfilterconf"
+
+  if [ $use_template -eq 1 ]
+  then
+    # Usage 2, templates
+    cp ../conf/f2b_jail.conf $finalfail2banjailconf
+    cp ../conf/f2b_filter.conf $finalfail2banfilterconf
+
+    if [ -n "${app:-}" ]
+    then
+      ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf"
+      ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf"
+    fi
+
+    # Replace all other variable given as arguments
+    for var_to_replace in ${others_var:-}; do
+      # ${var_to_replace^^} make the content of the variable on upper-cases
+      # ${!var_to_replace} get the content of the variable named $var_to_replace
+      ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf"
+      ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf"
+    done
+
+  else
+    # Usage 1, no template. Build a config file from scratch.
+    test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing."
+    test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing."
+
+    tee $finalfail2banjailconf <<EOF
+[$app]
+enabled = true
+port = $ports
+filter = $app
+logpath = $logpath
+maxretry = $max_retry
+EOF
+
+    tee $finalfail2banfilterconf <<EOF
+[INCLUDES]
+before = common.conf
+[Definition]
+failregex = $failregex
+ignoreregex =
+EOF
+  fi
+
+  # Common to usage 1 and 2.
+  ynh_store_file_checksum "$finalfail2banjailconf"
+  ynh_store_file_checksum "$finalfail2banfilterconf"
+
+  systemctl try-reload-or-restart fail2ban
+
+  local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")"
+  if [[ -n "$fail2ban_error" ]]; then
+    ynh_print_err --message="Fail2ban failed to load the jail for $app"
+    ynh_print_warn --message="${fail2ban_error#*WARNING}"
+  fi
+}
+
+# Remove the dedicated fail2ban config (jail and filter conf files)
+#
+# usage: ynh_remove_fail2ban_config
+ynh_remove_fail2ban_config () {
+  ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf"
+  ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf"
+  systemctl try-reload-or-restart fail2ban
+}

data/helpers.d/debug

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Debugger for app packagers
+#
+# usage: ynh_debug [--message=message] [--trace=1/0]
+# | arg: -m, --message= - The text to print
+# | arg: -t, --trace=   - Turn on or off the trace of the script. Usefull to trace nonly a small part of a script.
+ynh_debug () {
+	# Disable set xtrace for the helper itself, to not pollute the debug log
+	set +x
+	# Declare an array to define the options of this helper.
+	local legacy_args=mt
+	declare -Ar args_array=( [m]=message= [t]=trace= )
+	local message
+	local trace
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	# Redisable xtrace, ynh_handle_getopts_args set it back
+	set +x
+	message=${message:-}
+	trace=${trace:-}
+
+	if [ -n "$message" ]
+	then
+		ynh_print_log "\e[34m\e[1m[DEBUG]\e[0m ${message}" >&2
+	fi
+
+	if [ "$trace" == "1" ]
+	then
+		ynh_debug --message="Enable debugging"
+		set +x
+		# Get the current file descriptor of xtrace
+		old_bash_xtracefd=$BASH_XTRACEFD
+		# Add the current file name and the line number of any command currently running while tracing.
+		PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: '
+		# Force xtrace to stderr
+		BASH_XTRACEFD=2
+	fi
+	if [ "$trace" == "0" ]
+	then
+		ynh_debug --message="Disable debugging"
+		set +x
+		# Put xtrace back to its original fild descriptor
+		BASH_XTRACEFD=$old_bash_xtracefd
+	fi
+	# Renable set xtrace
+	set -x
+}
+
+# Execute a command and print the result as debug
+#
+# usage: ynh_debug_exec command to execute
+# usage: ynh_debug_exec "command to execute | following command"
+# In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe.
+#
+# | arg: command - command to execute
+ynh_debug_exec () {
+	ynh_debug --message="$(eval $@)"
+}

data/helpers.d/filesystem

@@ -1,3 +1,7 @@
+#!/bin/bash
+
+source /usr/share/yunohost/helpers.d/getopts
+
 CAN_BIND=${CAN_BIND:-1}
 
 # Add a file or a directory to the list of paths to backup
@@ -10,13 +14,13 @@ CAN_BIND=${CAN_BIND:-1}
 # 
 # If DEST is ended by a slash it complete this path with the basename of SRC.
 #
-# usage: ynh_backup src [dest [is_big [not_mandatory [arg]]]]
+# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory]
-# | arg: src - file or directory to bind or symlink or copy. it shouldn't be in
+# | arg: -s, --src_path - file or directory to bind or symlink or copy. it shouldn't be in
 # the backup dir. 
-# | arg: dest - destination file or directory inside the
+# | arg: -d, --dest_path - destination file or directory inside the
 # backup dir
-# | arg: is_big - 1 to indicate data are big (mail, video, image ...)
+# | arg: -b, --is_big - Indicate data are big (mail, video, image ...)
-# | arg: not_mandatory - 1 to indicate that if the file is missing, the backup can ignore it.
+# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it.
 # | arg: arg - Deprecated arg
 #
 # example:
@@ -44,16 +48,26 @@ CAN_BIND=${CAN_BIND:-1}
 #
 ynh_backup() {
     # TODO find a way to avoid injection by file strange naming !
-    local SRC_PATH="$1"
+
-    local DEST_PATH="${2:-}"
+    # Declare an array to define the options of this helper.
-    local IS_BIG="${3:-0}"
+    local legacy_args=sdbm
-    local NOT_MANDATORY="${4:-0}"
+    declare -Ar args_array=( [s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory )
+    local src_path
+    local dest_path
+    local is_big
+    local not_mandatory
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    local dest_path="${dest_path:-}"
+    local is_big="${is_big:-0}"
+    local not_mandatory="${not_mandatory:-0}"
+
     BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0}
 
     # If backing up core only (used by ynh_backup_before_upgrade),
     # don't backup big data items
-    if [ "$IS_BIG" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then
+    if [ "$is_big" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then
-      echo "$SRC_PATH will not be saved, because backup_core_only is set." >&2
+      echo "$src_path will not be saved, because backup_core_only is set." >&2
       return 0
     fi
     
@@ -61,15 +75,16 @@ ynh_backup() {
     # Format correctly source and destination paths
     # ==============================================================================
     # Be sure the source path is not empty
-    [[ -e "${SRC_PATH}" ]] || {
+    [[ -e "${src_path}" ]] || {
-        if [ "$NOT_MANDATORY" == "0" ]
+        echo "Source path '${src_path}' does not exist" >&2
+        if [ "$not_mandatory" == "0" ]
         then
 		echo "Source path '${SRC_PATH}' does not exist" >&2
 
                 # This is a temporary fix for fail2ban config files missing after the migration to stretch.
-                if echo "${SRC_PATH}" | grep --quiet "/etc/fail2ban"
+                if echo "${src_path}" | grep --quiet "/etc/fail2ban"
                 then
-                        touch "${SRC_PATH}"
+                        touch "${src_path}"
                         echo "The missing file will be replaced by a dummy one for the backup !!!" >&2
                 else
                         return 1
@@ -81,17 +96,17 @@ ynh_backup() {
 
     # Transform the source path as an absolute path
     # If it's a dir remove the ending /
-    SRC_PATH=$(realpath "$SRC_PATH")
+    src_path=$(realpath "$src_path")
 
     # If there is no destination path, initialize it with the source path
     # relative to "/".
-    # eg: SRC_PATH=/etc/yunohost -> DEST_PATH=etc/yunohost
+    # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost
-    if [[ -z "$DEST_PATH" ]]; then
+    if [[ -z "$dest_path" ]]; then
 
-        DEST_PATH="${SRC_PATH#/}"
+        dest_path="${src_path#/}"
 
     else
-        if [[ "${DEST_PATH:0:1}" == "/" ]]; then
+        if [[ "${dest_path:0:1}" == "/" ]]; then
 
             # If the destination path is an absolute path, transform it as a path
             # relative to the current working directory ($YNH_CWD)
@@ -100,43 +115,43 @@ ynh_backup() {
             # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/
             #
             # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR
-            DEST_PATH="${DEST_PATH#$YNH_CWD/}"
+            dest_path="${dest_path#$YNH_CWD/}"
 
             # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD
-            [[ "${DEST_PATH:0:1}" == "/" ]] \
+            [[ "${dest_path:0:1}" == "/" ]] \
-                && DEST_PATH="${DEST_PATH#/}"
+                && dest_path="${dest_path#/}"
         fi
 
-        # Complete DEST_PATH if ended by a /
+        # Complete dest_path if ended by a /
-        [[ "${DEST_PATH: -1}" == "/" ]] \
+        [[ "${dest_path: -1}" == "/" ]] \
-            &&  DEST_PATH="${DEST_PATH}/$(basename $SRC_PATH)"
+            &&  dest_path="${dest_path}/$(basename $src_path)"
     fi
 
-    # Check if DEST_PATH already exists in tmp archive
+    # Check if dest_path already exists in tmp archive
-    [[ ! -e "${DEST_PATH}" ]] || {
+    [[ ! -e "${dest_path}" ]] || {
-        echo "Destination path '${DEST_PATH}' already exist" >&2
+        echo "Destination path '${dest_path}' already exist" >&2
         return 1
     }
 
     # Add the relative current working directory to the destination path
-    local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR}"
+    local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}"
-    REL_DIR="${REL_DIR%/}/"
+    rel_dir="${rel_dir%/}/"
-    DEST_PATH="${REL_DIR}${DEST_PATH}"
+    dest_path="${rel_dir}${dest_path}"
-    DEST_PATH="${DEST_PATH#/}"
+    dest_path="${dest_path#/}"
     # ==============================================================================
 
     # ==============================================================================
     # Write file to backup into backup_list
     # ==============================================================================
-    local SRC=$(echo "${SRC_PATH}" | sed -r 's/"/\"\"/g')
+    local src=$(echo "${src_path}" | sed -r 's/"/\"\"/g')
-    local DEST=$(echo "${DEST_PATH}" | sed -r 's/"/\"\"/g')
+    local dest=$(echo "${dest_path}" | sed -r 's/"/\"\"/g')
-    echo "\"${SRC}\",\"${DEST}\"" >> "${YNH_BACKUP_CSV}"
+    echo "\"${src}\",\"${dest}\"" >> "${YNH_BACKUP_CSV}"
 
     # ==============================================================================
 
     # Create the parent dir of the destination path
     # It's for retro compatibility, some script consider ynh_backup creates this dir
-    mkdir -p $(dirname "$YNH_BACKUP_DIR/${DEST_PATH}")
+    mkdir -p $(dirname "$YNH_BACKUP_DIR/${dest_path}")
 }
 
 # Restore all files linked to the restore hook or to the restore app script
@@ -153,7 +168,7 @@ ynh_restore () {
     while read line; do
         local ORIGIN_PATH=$(echo "$line" | grep -ohP "^\"\K.*(?=\",\".*\"$)")
         local ARCHIVE_PATH=$(echo "$line" | grep -ohP "^\".*\",\"$REL_DIR\K.*(?=\"$)")
-        ynh_restore_file "$ARCHIVE_PATH" "$ORIGIN_PATH"
+        ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH"
     done
 }
 
@@ -183,13 +198,13 @@ with open(sys.argv[1], 'r') as backup_file:
 # Use the registered path in backup_list by ynh_backup to restore the file at
 # the good place.
 #
-# usage: ynh_restore_file ORIGIN_PATH [ DEST_PATH [NOT_MANDATORY]]
+# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory]
-# | arg: ORIGIN_PATH - Path where was located the file or the directory before
+# | arg: -o, --origin_path - Path where was located the file or the directory before
 # to be backuped or relative path to $YNH_CWD where it is located in the backup archive
-# | arg: DEST_PATH   - Path where restore the file or the dir, if unspecified,
+# | arg: -d, --dest_path - Path where restore the file or the dir, if unspecified,
 # the destination will be ORIGIN_PATH or if the ORIGIN_PATH doesn't exist in
 # the archive, the destination will be searched into backup.csv
-# | arg: NOT_MANDATORY - 1 to indicate that if the file is missing, the restore process can ignore it.
+# | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it.
 #
 # If DEST_PATH already exists and is lighter than 500 Mo, a backup will be made in 
 # /home/yunohost.conf/backup/. Otherwise, the existing file is removed.
@@ -205,49 +220,58 @@ with open(sys.argv[1], 'r') as backup_file:
 # ynh_restore_file "conf/nginx.conf"
 #
 ynh_restore_file () {
-    local ORIGIN_PATH="/${1#/}"
+    # Declare an array to define the options of this helper.
-    local ARCHIVE_PATH="$YNH_CWD${ORIGIN_PATH}"
+    local legacy_args=odm
-    # Default value for DEST_PATH = /$ORIGIN_PATH
+    declare -Ar args_array=( [o]=origin_path= [d]=dest_path= [m]=not_mandatory )
-    local DEST_PATH="${2:-$ORIGIN_PATH}"
+    local origin_path
-    local NOT_MANDATORY="${3:-0}"
+    local archive_path
+    local dest_path
+    local not_mandatory
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    local origin_path="/${origin_path#/}"
+    local archive_path="$YNH_CWD${origin_path}"
+    # Default value for dest_path = /$origin_path
+    local dest_path="${dest_path:-$origin_path}"
+    local not_mandatory="${not_mandatory:-0}"
 
-    # If ARCHIVE_PATH doesn't exist, search for a corresponding path in CSV
+    # If archive_path doesn't exist, search for a corresponding path in CSV
-    if [ ! -d "$ARCHIVE_PATH" ] && [ ! -f "$ARCHIVE_PATH" ] && [ ! -L "$ARCHIVE_PATH" ]; then
+    if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then
-        if [ "$NOT_MANDATORY" == "0" ]
+        if [ "$not_mandatory" == "0" ]
         then
-            ARCHIVE_PATH="$YNH_BACKUP_DIR/$(_get_archive_path \"$ORIGIN_PATH\")"
+            archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")"
         else
             return 0
         fi
     fi
 
     # Move the old directory if it already exists
-    if [[ -e "${DEST_PATH}" ]]
+    if [[ -e "${dest_path}" ]]
     then
         # Check if the file/dir size is less than 500 Mo
-        if [[ $(du -sb ${DEST_PATH} | cut -d"/" -f1) -le "500000000" ]]
+        if [[ $(du -sb ${dest_path} | cut -d"/" -f1) -le "500000000" ]]
         then
-            local backup_file="/home/yunohost.conf/backup/${DEST_PATH}.backup.$(date '+%Y%m%d.%H%M%S')"
+            local backup_file="/home/yunohost.conf/backup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
             mkdir -p "$(dirname "$backup_file")"
-            mv "${DEST_PATH}" "$backup_file"	# Move the current file or directory
+            mv "${dest_path}" "$backup_file"	# Move the current file or directory
         else
-            ynh_secure_remove ${DEST_PATH}
+            ynh_secure_remove --file=${dest_path}
         fi
     fi
 
-    # Restore ORIGIN_PATH into DEST_PATH
+    # Restore origin_path into dest_path
-    mkdir -p $(dirname "$DEST_PATH")
+    mkdir -p $(dirname "$dest_path")
 
     # Do a copy if it's just a mounting point
     if mountpoint -q $YNH_BACKUP_DIR; then
-        if [[ -d "${ARCHIVE_PATH}" ]]; then
+        if [[ -d "${archive_path}" ]]; then
-            ARCHIVE_PATH="${ARCHIVE_PATH}/."
+            archive_path="${archive_path}/."
-            mkdir -p "$DEST_PATH"
+            mkdir -p "$dest_path"
         fi
-        cp -a "$ARCHIVE_PATH" "${DEST_PATH}"
+        cp -a "$archive_path" "${dest_path}"
     # Do a move if YNH_BACKUP_DIR is already a copy
     else
-        mv "$ARCHIVE_PATH" "${DEST_PATH}"
+        mv "$archive_path" "${dest_path}"
     fi
 }
 
@@ -287,11 +311,28 @@ properly with chmod/chown." >&2
 #
 # $app should be defined when calling this helper
 #
-# usage: ynh_store_file_checksum file
+# usage: ynh_store_file_checksum --file=file
-# | arg: file - The file on which the checksum will performed, then stored.
+# | arg: -f, --file - The file on which the checksum will performed, then stored.
 ynh_store_file_checksum () {
-	local checksum_setting_name=checksum_${1//[\/ ]/_}	# Replace all '/' and ' ' by '_'
+	# Declare an array to define the options of this helper.
-	ynh_app_setting_set $app $checksum_setting_name $(sudo md5sum "$1" | cut -d' ' -f1)
+	local legacy_args=f
+	declare -Ar args_array=( [f]=file= )
+	local file
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	local checksum_setting_name=checksum_${file//[\/ ]/_}	# Replace all '/' and ' ' by '_'
+	ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1)
+
+	# If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
+	if [ -n "${backup_file_checksum-}" ]
+	then
+		# Print the diff between the previous file and the new one.
+		# diff return 1 if the files are different, so the || true
+		diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true
+	fi
+	# Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
+	unset backup_file_checksum
 }
 
 # Verify the checksum and backup the file if it's different
@@ -300,23 +341,31 @@ ynh_store_file_checksum () {
 #
 # $app should be defined when calling this helper
 #
-# usage: ynh_backup_if_checksum_is_different file
+# usage: ynh_backup_if_checksum_is_different --file=file
-# | arg: file - The file on which the checksum test will be perfomed.
+# | arg: -f, --file - The file on which the checksum test will be perfomed.
 #
 # | ret: Return the name a the backup file, or nothing
 ynh_backup_if_checksum_is_different () {
-	local file=$1
+	# Declare an array to define the options of this helper.
+	local legacy_args=f
+	declare -Ar args_array=( [f]=file= )
+	local file
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
 	local checksum_setting_name=checksum_${file//[\/ ]/_}	# Replace all '/' and ' ' by '_'
-	local checksum_value=$(ynh_app_setting_get $app $checksum_setting_name)
+	local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
+	# backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum
+	backup_file_checksum=""
 	if [ -n "$checksum_value" ]
 	then	# Proceed only if a value was stored into the app settings
 		if ! echo "$checksum_value $file" | sudo md5sum -c --status
 		then	# If the checksum is now different
-			local backup_file="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
+			backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
-			sudo mkdir -p "$(dirname "$backup_file")"
+			sudo mkdir -p "$(dirname "$backup_file_checksum")"
-			sudo cp -a "$file" "$backup_file"	# Backup the current file
+			sudo cp -a "$file" "$backup_file_checksum"	# Backup the current file
-			echo "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file" >&2
+			ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
-			echo "$backup_file"	# Return the name of the backup file
+			echo "$backup_file_checksum"	# Return the name of the backup file
 		fi
 	fi
 }
@@ -329,39 +378,51 @@ ynh_backup_if_checksum_is_different () {
 # | arg: -f, --file= - The file for which the checksum will be deleted
 ynh_delete_file_checksum () {
 	# Declare an array to define the options of this helper.
+	local legacy_args=f
 	declare -Ar args_array=( [f]=file= )
 	local file
 	# Manage arguments with getopts
 	ynh_handle_getopts_args "$@"
 
 	local checksum_setting_name=checksum_${file//[\/ ]/_}	# Replace all '/' and ' ' by '_'
-	ynh_app_setting_delete $app $checksum_setting_name
+	ynh_app_setting_delete --app=$app --key=$checksum_setting_name
 }
 
 # Remove a file or a directory securely
 #
-# usage: ynh_secure_remove path_to_remove
+# usage: ynh_secure_remove --file=path_to_remove
-# | arg: path_to_remove - File or directory to remove
+# | arg: -f, --file - File or directory to remove
 ynh_secure_remove () {
-	local path_to_remove=$1
+	# Declare an array to define the options of this helper.
+	local legacy_args=f
+	declare -Ar args_array=( [f]=file= )
+	local file
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
 	local forbidden_path=" \
 	/var/www \
 	/home/yunohost.app"
 
-	if [[ "$forbidden_path" =~ "$path_to_remove" \
+	if [ $# -ge 2 ]
+	then
+		echo "/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." >&2
+	fi
+
+	if [[ "$forbidden_path" =~ "$file" \
 		# Match all paths or subpaths in $forbidden_path
-		|| "$path_to_remove" =~ ^/[[:alnum:]]+$ \
+		|| "$file" =~ ^/[[:alnum:]]+$ \
 		# Match all first level paths from / (Like /var, /root, etc...)
-		|| "${path_to_remove:${#path_to_remove}-1}" = "/" ]]
+		|| "${file:${#file}-1}" = "/" ]]
 		# Match if the path finishes by /. Because it seems there is an empty variable
 	then
-		echo "Avoid deleting $path_to_remove." >&2
+		echo "Avoid deleting $file." >&2
 	else
-		if [ -e "$path_to_remove" ]
+		if [ -e "$file" ]
 		then
-			sudo rm -R "$path_to_remove"
+			sudo rm -R "$file"
 		else
-			echo "$path_to_remove wasn't deleted because it doesn't exist." >&2
+			echo "$file wasn't deleted because it doesn't exist." >&2
 		fi
 	fi
 }

data/helpers.d/getopts

@@ -98,10 +98,10 @@ ynh_handle_getopts_args () {
 
 				if [ "$parameter" = "?" ]
 				then
-					ynh_die "Invalid argument: -${OPTARG:-}"
+					ynh_die --message="Invalid argument: -${OPTARG:-}"
 				elif [ "$parameter" = ":" ]
 				then
-					ynh_die "-$OPTARG parameter requires an argument."
+					ynh_die --message="-$OPTARG parameter requires an argument."
 				else
 					local shift_value=1
 					# Use the long option, corresponding to the short option read by getopts, as a variable
@@ -150,6 +150,9 @@ ynh_handle_getopts_args () {
 									# If there's already another value for this option, add a ; before adding the new value
 									eval ${option_var}+="\;"
 								fi
+								# Escape double quote to prevent any interpretation during the eval
+								all_args[$i]="${all_args[$i]//\"/\\\"}"
+
 								eval ${option_var}+=\"${all_args[$i]}\"
 								shift_value=$(( shift_value + 1 ))
 							fi
@@ -179,7 +182,7 @@ ynh_handle_getopts_args () {
 				# Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument.
 				option_flag=${getopts_parameters:$i:1}
 				if [ -z "$option_flag" ]; then
-						ynh_print_warn "Too many arguments ! \"${arguments[$i]}\" will be ignored."
+						ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored."
 						continue
 				fi
 				# Use the long option, corresponding to the option_flag, as a variable
@@ -188,6 +191,9 @@ ynh_handle_getopts_args () {
 				# The variable name will be stored in 'option_var'
 				local option_var="${args_array[$option_flag]%=}"
 
+				# Escape double quote to prevent any interpretation during the eval
+				arguments[$i]="${arguments[$i]//\"/\\\"}"
+
 				# Store each value given as argument in the corresponding variable
 				# The values will be stored in the same order than $args_array
 				eval ${option_var}+=\"${arguments[$i]}\"

data/helpers.d/ip

@@ -1,6 +1,8 @@
+#!/bin/bash
+
 # Validate an IP address
 #
-# usage: ynh_validate_ip [family] [ip_address]
+# usage: ynh_validate_ip --family=family --ip_address=ip_address
 # | ret: 0 for valid ip addresses, 1 otherwise
 #
 # example: ynh_validate_ip 4 111.222.333.444
@@ -9,17 +11,22 @@ ynh_validate_ip()
 {
   # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298
 
-  local IP_ADDRESS_FAMILY=$1
+  # Declare an array to define the options of this helper.
-  local IP_ADDRESS=$2
+  local legacy_args=fi
+  declare -Ar args_array=( [f]=family= [i]=ip_address= )
+  local family
+  local ip_address
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
 
-  [ "$IP_ADDRESS_FAMILY" == "4" ] || [ "$IP_ADDRESS_FAMILY" == "6" ] || return 1
+  [ "$family" == "4" ] || [ "$family" == "6" ] || return 1
 
   python /dev/stdin << EOF
 import socket
 import sys
 family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 }
 try:
-    socket.inet_pton(family["$IP_ADDRESS_FAMILY"], "$IP_ADDRESS")
+    socket.inet_pton(family["$family"], "$ip_address")
 except socket.error:
     sys.exit(1)
 sys.exit(0)
@@ -30,12 +37,19 @@ EOF
 #
 # example: ynh_validate_ip4 111.222.333.444
 #
-# usage: ynh_validate_ip4 <ip_address>
+# usage: ynh_validate_ip4 --ip_address=ip_address
 # | ret: 0 for valid ipv4 addresses, 1 otherwise
 #
 ynh_validate_ip4()
 {
-  ynh_validate_ip 4 $1
+  # Declare an array to define the options of this helper.
+  local legacy_args=i
+  declare -Ar args_array=( [i]=ip_address= )
+  local ip_address
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
+
+  ynh_validate_ip 4 $ip_address
 }
 
 
@@ -43,10 +57,17 @@ ynh_validate_ip4()
 #
 # example: ynh_validate_ip6 2000:dead:beef::1
 #
-# usage: ynh_validate_ip6 <ip_address>
+# usage: ynh_validate_ip6 --ip_address=ip_address
 # | ret: 0 for valid ipv6 addresses, 1 otherwise
 #
 ynh_validate_ip6()
 {
-  ynh_validate_ip 6 $1
+  # Declare an array to define the options of this helper.
+  local legacy_args=i
+  declare -Ar args_array=( [i]=ip_address= )
+  local ip_address
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
+
+  ynh_validate_ip 6 $ip_address
 }

data/helpers.d/mysql

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql
 
 # Open a connection as a user
@@ -5,32 +7,60 @@ MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql
 # example: ynh_mysql_connect_as 'user' 'pass' <<< "UPDATE ...;"
 # example: ynh_mysql_connect_as 'user' 'pass' < /path/to/file.sql
 #
-# usage: ynh_mysql_connect_as user pwd [db]
+# usage: ynh_mysql_connect_as --user=user --password=password [--database=database]
-# | arg: user - the user name to connect as
+# | arg: -u, --user - the user name to connect as
-# | arg: pwd - the user password
+# | arg: -p, --password - the user password
-# | arg: db - the database to connect to
+# | arg: -d, --database - the database to connect to
 ynh_mysql_connect_as() {
-    mysql -u "$1" --password="$2" -B "${3:-}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=upd
+    declare -Ar args_array=( [u]=user= [p]=password= [d]=database= )
+    local user
+    local password
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    database="${database:-}"
+
+    mysql -u "$user" --password="$password" -B "$database"
 }
 
 # Execute a command as root user
 #
-# usage: ynh_mysql_execute_as_root sql [db]
+# usage: ynh_mysql_execute_as_root --sql=sql [--database=database]
-# | arg: sql - the SQL command to execute
+# | arg: -s, --sql - the SQL command to execute
-# | arg: db - the database to connect to
+# | arg: -d, --database - the database to connect to
 ynh_mysql_execute_as_root() {
-    ynh_mysql_connect_as "root" "$(sudo cat $MYSQL_ROOT_PWD_FILE)" \
+    # Declare an array to define the options of this helper.
-        "${2:-}" <<< "$1"
+    local legacy_args=sd
+    declare -Ar args_array=( [s]=sql= [d]=database= )
+    local sql
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    database="${database:-}"
+
+    ynh_mysql_connect_as --user="root" --password="$(sudo cat $MYSQL_ROOT_PWD_FILE)" \
+        --database="$database" <<< "$sql"
 }
 
 # Execute a command from a file as root user
 #
-# usage: ynh_mysql_execute_file_as_root file [db]
+# usage: ynh_mysql_execute_file_as_root --file=file [--database=database]
-# | arg: file - the file containing SQL commands
+# | arg: -f, --file - the file containing SQL commands
-# | arg: db - the database to connect to
+# | arg: -d, --database - the database to connect to
 ynh_mysql_execute_file_as_root() {
-    ynh_mysql_connect_as "root" "$(sudo cat $MYSQL_ROOT_PWD_FILE)" \
+    # Declare an array to define the options of this helper.
-        "${2:-}" < "$1"
+    local legacy_args=fd
+    declare -Ar args_array=( [f]=file= [d]=database= )
+    local file
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    database="${database:-}"
+
+    ynh_mysql_connect_as --user="root" --password="$(sudo cat $MYSQL_ROOT_PWD_FILE)" \
+        --database="$database" < "$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user
@@ -53,7 +83,7 @@ ynh_mysql_create_db() {
         sql+=" WITH GRANT OPTION;"
     fi
 
-    ynh_mysql_execute_as_root "$sql"
+    ynh_mysql_execute_as_root --sql="$sql"
 }
 
 # Drop a database
@@ -66,18 +96,25 @@ ynh_mysql_create_db() {
 # usage: ynh_mysql_drop_db db
 # | arg: db - the database name to drop
 ynh_mysql_drop_db() {
-    ynh_mysql_execute_as_root "DROP DATABASE ${1};"
+    ynh_mysql_execute_as_root --sql="DROP DATABASE ${1};"
 }
 
 # Dump a database
 #
 # example: ynh_mysql_dump_db 'roundcube' > ./dump.sql
 #
-# usage: ynh_mysql_dump_db db
+# usage: ynh_mysql_dump_db --database=database
-# | arg: db - the database name to dump
+# | arg: -d, --database - the database name to dump
 # | ret: the mysqldump output
 ynh_mysql_dump_db() {
-    mysqldump -u "root" -p"$(sudo cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$1"
+    # Declare an array to define the options of this helper.
+    local legacy_args=d
+    declare -Ar args_array=( [d]=database= )
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    mysqldump -u "root" -p"$(sudo cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$database"
 }
 
 # Create a user
@@ -89,17 +126,23 @@ ynh_mysql_dump_db() {
 # | arg: pwd - the password to identify user by
 ynh_mysql_create_user() {
     ynh_mysql_execute_as_root \
-        "CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';"
+        --sql="CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';"
 }
 
 # Check if a mysql user exists
 #
-# usage: ynh_mysql_user_exists user
+# usage: ynh_mysql_user_exists --user=user
-# | arg: user - the user for which to check existence
+# | arg: -u, --user - the user for which to check existence
 ynh_mysql_user_exists()
 {
-   local user=$1
+   # Declare an array to define the options of this helper.
-   if [[ -z $(ynh_mysql_execute_as_root "SELECT User from mysql.user WHERE User = '$user';") ]]
+   local legacy_args=u
+   declare -Ar args_array=( [u]=user= )
+   local user
+   # Manage arguments with getopts
+   ynh_handle_getopts_args "$@"
+
+   if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]
    then
       return 1
    else
@@ -114,7 +157,7 @@ ynh_mysql_user_exists()
 # usage: ynh_mysql_drop_user user
 # | arg: user - the user name to drop
 ynh_mysql_drop_user() {
-    ynh_mysql_execute_as_root "DROP USER '${1}'@'localhost';"
+    ynh_mysql_execute_as_root --sql="DROP USER '${1}'@'localhost';"
 }
 
 # Create a database, an user and its password. Then store the password in the app's config
@@ -122,28 +165,42 @@ ynh_mysql_drop_user() {
 # After executing this helper, the password of the created database will be available in $db_pwd
 # It will also be stored as "mysqlpwd" into the app settings.
 #
-# usage: ynh_mysql_setup_db user name [pwd]
+# usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
-# | arg: user - Owner of the database
+# | arg: -u, --db_user - Owner of the database
-# | arg: name - Name of the database
+# | arg: -n, --db_name - Name of the database
-# | arg: pwd - Password of the database. If not given, a password will be generated
+# | arg: -p, --db_pwd - Password of the database. If not given, a password will be generated
 ynh_mysql_setup_db () {
-	local db_user="$1"
+	# Declare an array to define the options of this helper.
-	local db_name="$2"
+	local legacy_args=unp
+	declare -Ar args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= )
+	local db_user
+	local db_name
+	db_pwd=""
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
 	local new_db_pwd=$(ynh_string_random)	# Generate a random password
-	# If $3 is not given, use new_db_pwd instead for db_pwd.
+	# If $db_pwd is not given, use new_db_pwd instead for db_pwd
-	db_pwd="${3:-$new_db_pwd}"
+	db_pwd="${db_pwd:-$new_db_pwd}"
+
 	ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd"	# Create the database
-	ynh_app_setting_set $app mysqlpwd $db_pwd	# Store the password in the app's config
+	ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd	# Store the password in the app's config
 }
 
 # Remove a database if it exists, and the associated user
 #
-# usage: ynh_mysql_remove_db user name
+# usage: ynh_mysql_remove_db --db_user=user --db_name=name
-# | arg: user - Owner of the database
+# | arg: -u, --db_user - Owner of the database
-# | arg: name - Name of the database
+# | arg: -n, --db_name - Name of the database
 ynh_mysql_remove_db () {
-	local db_user="$1"
+	# Declare an array to define the options of this helper.
-	local db_name="$2"
+	local legacy_args=un
+	declare -Ar args_array=( [u]=db_user= [n]=db_name= )
+	local db_user
+	local db_name
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
 	local mysql_root_password=$(sudo cat $MYSQL_ROOT_PWD_FILE)
 	if mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name"; then	# Check if the database exists
 		echo "Removing database $db_name" >&2
@@ -153,7 +210,7 @@ ynh_mysql_remove_db () {
 	fi
 
 	# Remove mysql user if it exists
-        if $(ynh_mysql_user_exists $db_user); then
+        if $(ynh_mysql_user_exists --user=$db_user); then
 		ynh_mysql_drop_user $db_user
 	fi
 }
@@ -163,10 +220,17 @@ ynh_mysql_remove_db () {
 #
 # example: dbname=$(ynh_sanitize_dbid $app)
 #
-# usage: ynh_sanitize_dbid name
+# usage: ynh_sanitize_dbid --db_name=name
-# | arg: name - name to correct/sanitize
+# | arg: -n, --db_name - name to correct/sanitize
 # | ret: the corrected name
 ynh_sanitize_dbid () {
-	local dbid=${1//[-.]/_}	# We should avoid having - and . in the name of databases. They are replaced by _
+	# Declare an array to define the options of this helper.
-	echo $dbid
+	local legacy_args=n
+	declare -Ar args_array=( [n]=db_name= )
+	local db_name
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	# We should avoid having - and . in the name of databases. They are replaced by _
+	echo ${db_name//[-.]/_}
 }

data/helpers.d/network

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Normalize the url path syntax
 # Handle the slash at the beginning of path and its absence at ending
 # Return a normalized url path
@@ -8,11 +10,17 @@
 #          ynh_normalize_url_path /example/ -> /example
 #          ynh_normalize_url_path / -> /
 #
-# usage: ynh_normalize_url_path path_to_normalize
+# usage: ynh_normalize_url_path --path_url=path_to_normalize
-# | arg: url_path_to_normalize - URL path to normalize before using it
+# | arg: -p, --path_url - URL path to normalize before using it
 ynh_normalize_url_path () {
-	local path_url=$1
+	# Declare an array to define the options of this helper.
-	test -n "$path_url" || ynh_die "ynh_normalize_url_path expect a URL path as first argument and received nothing."
+	local legacy_args=p
+	declare -Ar args_array=( [p]=path_url= )
+	local path_url
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing."
 	if [ "${path_url:0:1}" != "/" ]; then    # If the first character is not a /
 		path_url="/$path_url"    # Add / at begin of path variable
 	fi
@@ -24,13 +32,19 @@ ynh_normalize_url_path () {
 
 # Find a free port and return it
 #
-# example: port=$(ynh_find_port 8080)
+# example: port=$(ynh_find_port --port=8080)
 #
-# usage: ynh_find_port begin_port
+# usage: ynh_find_port --port=begin_port
-# | arg: begin_port - port to start to search
+# | arg: -p, --port - port to start to search
 ynh_find_port () {
-	local port=$1
+	# Declare an array to define the options of this helper.
-	test -n "$port" || ynh_die "The argument of ynh_find_port must be a valid port."
+	local legacy_args=p
+	declare -Ar args_array=( [p]=port= )
+	local port
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port."
 	while netcat -z 127.0.0.1 $port       # Check if the port is free
 	do
 		port=$((port+1))	# Else, pass to next port
@@ -40,28 +54,40 @@ ynh_find_port () {
 
 # Check availability of a web path
 #
-# example: ynh_webpath_available some.domain.tld /coffee
+# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee
 #
-# usage: ynh_webpath_available domain path
+# usage: ynh_webpath_available --domain=domain --path_url=path
-# | arg: domain - the domain/host of the url
+# | arg: -d, --domain - the domain/host of the url
-# | arg: path - the web path to check the availability of
+# | arg: -p, --path_url - the web path to check the availability of
 ynh_webpath_available () {
-	local domain=$1
+	# Declare an array to define the options of this helper.
-	local path=$2
+	local legacy_args=dp
-	sudo yunohost domain url-available $domain $path
+	declare -Ar args_array=( [d]=domain= [p]=path_url= )
+	local domain
+	local path_url
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	sudo yunohost domain url-available $domain $path_url
 }
 
 # Register/book a web path for an app
 #
-# example: ynh_webpath_register wordpress some.domain.tld /coffee
+# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee
 #
-# usage: ynh_webpath_register app domain path
+# usage: ynh_webpath_register --app=app --domain=domain --path_url=path
-# | arg: app - the app for which the domain should be registered
+# | arg: -a, --app - the app for which the domain should be registered
-# | arg: domain - the domain/host of the web path
+# | arg: -d, --domain - the domain/host of the web path
-# | arg: path - the web path to be registered
+# | arg: -p, --path_url - the web path to be registered
 ynh_webpath_register () {
-	local app=$1
+	# Declare an array to define the options of this helper.
-	local domain=$2
+	local legacy_args=adp
-	local path=$3
+	declare -Ar args_array=( [a]=app= [d]=domain= [p]=path_url= )
-	sudo yunohost app register-url $app $domain $path
+	local app
+	local domain
+	local path_url
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	sudo yunohost app register-url $app $domain $path_url
 }

data/helpers.d/nodejs

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 n_install_dir="/opt/node_n"
 node_version_path="$n_install_dir/n/versions/node"
 # N_PREFIX is the directory of n, it needs to be loaded as a environment variable.
@@ -15,7 +17,7 @@ ynh_install_n () {
 	echo "SOURCE_URL=https://github.com/tj/n/archive/v2.1.7.tar.gz
 SOURCE_SUM=2ba3c9d4dd3c7e38885b37e02337906a1ee91febe6d5c9159d89a9050f2eea8f" > "../conf/n.src"
 	# Download and extract n
-	ynh_setup_source "$n_install_dir/git" n
+	ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n
 	# Install n
 	(cd "$n_install_dir/git"
 	PREFIX=$N_PREFIX make install 2>&1)
@@ -35,7 +37,7 @@ SOURCE_SUM=2ba3c9d4dd3c7e38885b37e02337906a1ee91febe6d5c9159d89a9050f2eea8f" > "
 #
 # usage: ynh_use_nodejs
 ynh_use_nodejs () {
-	nodejs_version=$(ynh_app_setting_get $app nodejs_version)
+	nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
 	nodejs_use_version="echo \"Deprecated command, should be removed\""
 
@@ -53,13 +55,19 @@ ynh_use_nodejs () {
 #
 # ynh_install_nodejs will install the version of node provided as argument by using n.
 #
-# usage: ynh_install_nodejs [nodejs_version]
+# usage: ynh_install_nodejs --nodejs_version=nodejs_version
-# | arg: nodejs_version - Version of node to install.
+# | arg: -n, --nodejs_version - Version of node to install.
 #        If possible, prefer to use major version number (e.g. 8 instead of 8.10.0).
 #        The crontab will handle the update of minor versions when needed.
 ynh_install_nodejs () {
 	# Use n, https://github.com/tj/n to manage the nodejs versions
-	nodejs_version="$1"
+
+	# Declare an array to define the options of this helper.
+	local legacy_args=n
+	declare -Ar args_array=( [n]=nodejs_version= )
+	local nodejs_version
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
 
 	# Create $n_install_dir
 	mkdir -p "$n_install_dir"
@@ -80,7 +88,7 @@ ynh_install_nodejs () {
 	fi
 
 	# Modify the default N_PREFIX in n script
-	ynh_replace_string "^N_PREFIX=\${N_PREFIX-.*}$" "N_PREFIX=\${N_PREFIX-$N_PREFIX}" "$n_install_dir/bin/n"
+	ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n"
 
 	# Restore /usr/local/bin in PATH
 	PATH=$CLEAR_PATH
@@ -90,7 +98,13 @@ ynh_install_nodejs () {
 	test -x /usr/bin/npm_n && mv /usr/bin/npm_n /usr/bin/npm
 
 	# Install the requested version of nodejs
-	n $nodejs_version
+	uname=$(uname -m)
+	if [[ $uname =~ aarch64 ||  $uname =~ arm64]]
+	then
+		n $nodejs_version --arch=arm64
+	else
+		n $nodejs_version
+	fi
 
 	# Find the last "real" version for this major version of node.
 	real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1)
@@ -106,7 +120,7 @@ ynh_install_nodejs () {
 	echo "$YNH_APP_ID:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
 
 	# Store nodejs_version into the config of this app
-	ynh_app_setting_set $app nodejs_version $nodejs_version
+	ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version
 
 	# Build the update script and set the cronjob
 	ynh_cron_upgrade_node
@@ -122,7 +136,7 @@ ynh_install_nodejs () {
 #
 # usage: ynh_remove_nodejs
 ynh_remove_nodejs () {
-	nodejs_version=$(ynh_app_setting_get $app nodejs_version)
+	nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
 	# Remove the line for this app
 	sed --in-place "/$YNH_APP_ID:$nodejs_version/d" "$n_install_dir/ynh_app_version"
@@ -136,8 +150,8 @@ ynh_remove_nodejs () {
 	# If no other app uses n, remove n
 	if [ ! -s "$n_install_dir/ynh_app_version" ]
 	then
-		ynh_secure_remove "$n_install_dir"
+		ynh_secure_remove --file="$n_install_dir"
-		ynh_secure_remove "/usr/local/n"
+		ynh_secure_remove --file="/usr/local/n"
 		sed --in-place "/N_PREFIX/d" /root/.bashrc
 		rm -f /etc/cron.daily/node_update
 	fi

data/helpers.d/package

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Check if apt is free to use, or wait, until timeout.
 #
 # [internal]
@@ -15,6 +17,21 @@ ynh_wait_dpkg_free() {
             # Sleep an exponential time at each round
             sleep $(( try * try ))
         else
+            # Check if dpkg hasn't been interrupted and is fully available.
+            # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174
+            local dpkg_dir="/var/lib/dpkg/updates/"
+
+            # For each file in $dpkg_dir
+            while read dpkg_file <&9
+            do
+                # Check if the name of this file contains only numbers.
+                if echo "$dpkg_file" | grep -Pq "^[[:digit:]]*$"
+                then
+                    # If so, that a remaining of dpkg.
+                    ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem."
+                    return 1
+                fi
+            done 9<<< "$(ls -1 $dpkg_dir)"
             return 0
         fi
     done
@@ -23,26 +40,40 @@ ynh_wait_dpkg_free() {
 
 # Check either a package is installed or not
 #
-# example: ynh_package_is_installed 'yunohost' && echo "ok"
+# example: ynh_package_is_installed --package=yunohost && echo "ok"
 #
-# usage: ynh_package_is_installed name
+# usage: ynh_package_is_installed --package=name
-# | arg: name - the package name to check
+# | arg: -p, --package - the package name to check
 ynh_package_is_installed() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=p
+    declare -Ar args_array=( [p]=package= )
+    local package
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
     ynh_wait_dpkg_free
-    dpkg-query -W -f '${Status}' "$1" 2>/dev/null \
+    dpkg-query -W -f '${Status}' "$package" 2>/dev/null \
         | grep -c "ok installed" &>/dev/null
 }
 
 # Get the version of an installed package
 #
-# example: version=$(ynh_package_version 'yunohost')
+# example: version=$(ynh_package_version --package=yunohost)
 #
-# usage: ynh_package_version name
+# usage: ynh_package_version --package=name
-# | arg: name - the package name to get version
+# | arg: -p, --package - the package name to get version
 # | ret: the version or an empty string
 ynh_package_version() {
-    if ynh_package_is_installed "$1"; then
+    # Declare an array to define the options of this helper.
-        dpkg-query -W -f '${Version}' "$1" 2>/dev/null
+    local legacy_args=p
+    declare -Ar args_array=( [p]=package= )
+    local package
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    if ynh_package_is_installed "$package"; then
+        dpkg-query -W -f '${Version}' "$package" 2>/dev/null
     else
         echo ''
     fi
@@ -55,7 +86,7 @@ ynh_package_version() {
 # usage: ynh_apt update
 ynh_apt() {
     ynh_wait_dpkg_free
-    DEBIAN_FRONTEND=noninteractive sudo apt-get -y $@
+    DEBIAN_FRONTEND=noninteractive apt-get -y $@
 }
 
 # Update package index files
@@ -135,7 +166,7 @@ ynh_package_install_from_equivs () {
     (cd "$TMPDIR"
     equivs-build ./control 1> /dev/null
     dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1)
-    ynh_package_install -f || ynh_die "Unable to install dependencies"
+    ynh_package_install -f || ynh_die --message="Unable to install dependencies"
     [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR	# Remove the temp dir.
 
     # check if the package is actually installed
@@ -176,9 +207,9 @@ Description: Fake package for ${app} (YunoHost app) dependencies
  This meta-package is only responsible of installing its dependencies.
 EOF
     ynh_package_install_from_equivs /tmp/${dep_app}-ynh-deps.control \
-        || ynh_die "Unable to install dependencies"	# Install the fake package and its dependencies
+        || ynh_die --message="Unable to install dependencies"	# Install the fake package and its dependencies
     rm /tmp/${dep_app}-ynh-deps.control
-    ynh_app_setting_set $app apt_dependencies $dependencies
+    ynh_app_setting_set --app=$app --key=apt_dependencies --value="$dependencies"
 }
 
 # Remove fake package and its dependencies

data/helpers.d/print

@@ -1,15 +1,32 @@
+#!/bin/bash
+
 # Print a message to stderr and exit
-# usage: ynh_die MSG [RETCODE]
+# usage: ynh_die --message=MSG [--ret_code=RETCODE]
 ynh_die() {
-  echo "$1" 1>&2
+  # Declare an array to define the options of this helper.
-  exit "${2:-1}"
+  local legacy_args=mc
+  declare -Ar args_array=( [m]=message= [c]=ret_code= )
+  local message
+  local ret_code
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
+
+  echo "$message" 1>&2
+  exit "${ret_code:-1}"
 }
 
 # Display a message in the 'INFO' logging category
 #
-# usage: ynh_print_info "Some message"
+# usage: ynh_print_info --message="Some message"
 ynh_print_info() {
-    echo "$1" >> "$YNH_STDINFO"
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    declare -Ar args_array=( [m]=message= )
+    local message
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    echo "$message" >> "$YNH_STDINFO"
 }
 
 # Ignore the yunohost-cli log to prevent errors with conditional commands
@@ -39,18 +56,32 @@ ynh_print_log () {
 
 # Print a warning on stderr
 #
-# usage: ynh_print_warn "Text to print"
+# usage: ynh_print_warn --message="Text to print"
-# | arg: text - The text to print
+# | arg: -m, --message - The text to print
 ynh_print_warn () {
-  ynh_print_log "\e[93m\e[1m[WARN]\e[0m ${1}" >&2
+  # Declare an array to define the options of this helper.
+  local legacy_args=m
+  declare -Ar args_array=( [m]=message= )
+  local message
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
+
+  ynh_print_log "\e[93m\e[1m[WARN]\e[0m ${message}" >&2
 }
 
 # Print an error on stderr
 #
-# usage: ynh_print_err "Text to print"
+# usage: ynh_print_err --message="Text to print"
-# | arg: text - The text to print
+# | arg: -m, --message - The text to print
 ynh_print_err () {
-  ynh_print_log "\e[91m\e[1m[ERR]\e[0m ${1}" >&2
+  # Declare an array to define the options of this helper.
+  local legacy_args=m
+  declare -Ar args_array=( [m]=message= )
+  local message
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
+
+  ynh_print_log "\e[91m\e[1m[ERR]\e[0m ${message}" >&2
 }
 
 # Execute a command and print the result as an error
@@ -124,3 +155,81 @@ ynh_print_ON () {
 	# Print an echo only for the log, to be able to know that ynh_print_ON has been called.
 	echo ynh_print_ON > /dev/null
 }
+
+# Print a message as INFO and show progression during an app script
+#
+# usage: ynh_script_progression --message=message [--weight=weight] [--time]
+# | arg: -m, --message= - The text to print
+# | arg: -w, --weight=  - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script.
+# | arg: -t, --time=    - Print the execution time since the last call to this helper. Especially usefull to define weights.
+# | arg: -l, --last=    - Use for the last call of the helper, to fill te progression bar.
+increment_progression=0
+previous_weight=0
+# Define base_time when the file is sourced
+base_time=$(date +%s)
+ynh_script_progression () {
+	# Declare an array to define the options of this helper.
+	declare -Ar args_array=( [m]=message= [w]=weight= [t]=time [l]=last )
+	local message
+	local weight
+	local time
+	local last
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	weight=${weight:-1}
+	time=${time:-0}
+	last=${last:-0}
+
+	# Get execution time since the last $base_time
+	local exec_time=$(( $(date +%s) - $base_time ))
+	base_time=$(date +%s)
+
+	# Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented.
+	local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)"
+	# Get the number of call with a weight value
+	local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0)
+
+	# Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight
+	local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]].*\)/\1/g')"
+	# Get the weight of each occurrences of 'ynh_script_progression' in the script using -w
+	local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]].*\)/\1/g')"
+	# Each value will be on a different line.
+	# Remove each 'end of line' and replace it by a '+' to sum the values.
+	local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 ))
+
+	# max_progression is a total number of calls to this helper.
+	#    Less the number of calls with a weight value.
+	#    Plus the total of weight values
+	local max_progression=$(( $helper_calls - $weight_calls + $weight_values ))
+
+	# Increment each execution of ynh_script_progression in this script by the weight of the previous call.
+	increment_progression=$(( $increment_progression + $previous_weight ))
+	# Store the weight of the current call in $previous_weight for next call
+	previous_weight=$weight
+
+	# Set the scale of the progression bar
+	local scale=20
+	# progress_string(1,2) should have the size of the scale.
+	local progress_string1="####################"
+	local progress_string0="...................."
+
+	# Reduce $increment_progression to the size of the scale
+	if [ $last -eq 0 ]
+	then
+		local effective_progression=$(( $increment_progression * $scale / $max_progression ))
+	# If last is specified, fill immediately the progression_bar
+	else
+		local effective_progression=$scale
+	fi
+
+	# Build $progression_bar from progress_string(1,2) according to $effective_progression
+	local progression_bar="${progress_string1:0:$effective_progression}${progress_string0:0:$(( $scale - $effective_progression ))}"
+
+	local print_exec_time=""
+	if [ $time -eq 1 ]
+	then
+		print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]"
+	fi
+
+	ynh_print_info "[$progression_bar] > ${message}${print_exec_time}"
+}

data/helpers.d/setting

@@ -1,27 +1,54 @@
+#!/bin/bash
+
 # Get an application setting
 #
-# usage: ynh_app_setting_get app key
+# usage: ynh_app_setting_get --app=app --key=key
-# | arg: app - the application id
+# | arg: -a, --app - the application id
-# | arg: key - the setting to get
+# | arg: -k, --key - the setting to get
 ynh_app_setting_get() {
-    sudo yunohost app setting "$1" "$2" --output-as plain --quiet
+    # Declare an array to define the options of this helper.
+    local legacy_args=ak
+    declare -Ar args_array=( [a]=app= [k]=key= )
+    local app
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    sudo yunohost app setting "$app" "$key" --output-as plain --quiet
 }
 
 # Set an application setting
 #
-# usage: ynh_app_setting_set app key value
+# usage: ynh_app_setting_set --app=app --key=key --value=value
-# | arg: app - the application id
+# | arg: -a, --app - the application id
-# | arg: key - the setting name to set
+# | arg: -k, --key - the setting name to set
-# | arg: value - the setting value to set
+# | arg: -v, --value - the setting value to set
 ynh_app_setting_set() {
-    sudo yunohost app setting "$1" "$2" --value="$3" --quiet
+    # Declare an array to define the options of this helper.
+    local legacy_args=akv
+    declare -Ar args_array=( [a]=app= [k]=key= [v]=value= )
+    local app
+    local key
+    local value
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    sudo yunohost app setting "$app" "$key" --value="$value" --quiet
 }
 
 # Delete an application setting
 #
-# usage: ynh_app_setting_delete app key
+# usage: ynh_app_setting_delete --app=app --key=key
-# | arg: app - the application id
+# | arg: -a, --app - the application id
-# | arg: key - the setting to delete
+# | arg: -k, --key - the setting to delete
 ynh_app_setting_delete() {
-    sudo yunohost app setting -d "$1" "$2" --quiet
+    # Declare an array to define the options of this helper.
+    local legacy_args=ak
+    declare -Ar args_array=( [a]=app= [k]=key= )
+    local app
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    sudo yunohost app setting -d "$app" "$key" --quiet
 }

data/helpers.d/string

@@ -1,59 +1,79 @@
+#!/bin/bash
+
 # Generate a random string
 #
-# example: pwd=$(ynh_string_random 8)
+# example: pwd=$(ynh_string_random --length=8)
 #
-# usage: ynh_string_random [length]
+# usage: ynh_string_random [--length=string_length]
-# | arg: length - the string length to generate (default: 24)
+# | arg: -l, --length - the string length to generate (default: 24)
 ynh_string_random() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=l
+    declare -Ar args_array=( [l]=length= )
+    local length
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    length=${length:-24}
+
     dd if=/dev/urandom bs=1 count=1000 2> /dev/null \
       | tr -c -d 'A-Za-z0-9' \
-      | sed -n 's/\(.\{'"${1:-24}"'\}\).*/\1/p'
+      | sed -n 's/\(.\{'"$length"'\}\).*/\1/p'
 }
 
 # Substitute/replace a string (or expression) by another in a file
 #
-# usage: ynh_replace_string match_string replace_string target_file
+# usage: ynh_replace_string --match_string=match_string --replace_string=replace_string --target_file=target_file
-# | arg: match_string - String to be searched and replaced in the file
+# | arg: -m, --match_string - String to be searched and replaced in the file
-# | arg: replace_string - String that will replace matches
+# | arg: -r, --replace_string - String that will replace matches
-# | arg: target_file - File in which the string will be replaced.
+# | arg: -f, --target_file - File in which the string will be replaced.
 #
 # As this helper is based on sed command, regular expressions and
 # references to sub-expressions can be used
 # (see sed manual page for more information)
 ynh_replace_string () {
+	# Declare an array to define the options of this helper.
+	local legacy_args=mrf
+	declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
+	local match_string
+	local replace_string
+	local target_file
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
 	local delimit=@
-	local match_string=$1
-	local replace_string=$2
-	local workfile=$3
-  
 	# Escape the delimiter if it's in the string.
 	match_string=${match_string//${delimit}/"\\${delimit}"}
 	replace_string=${replace_string//${delimit}/"\\${delimit}"}
 
-	sudo sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$workfile"
+	sudo sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file"
 }
 
 # Substitute/replace a special string by another in a file
 #
-# usage: ynh_replace_special_string match_string replace_string target_file
+# usage: ynh_replace_special_string --match_string=match_string --replace_string=replace_string --target_file=target_file
-# | arg: match_string - String to be searched and replaced in the file
+# | arg: -m, --match_string - String to be searched and replaced in the file
-# | arg: replace_string - String that will replace matches
+# | arg: -r, --replace_string - String that will replace matches
-# | arg: target_file - File in which the string will be replaced.
+# | arg: -t, --target_file - File in which the string will be replaced.
 #
 # This helper will use ynh_replace_string, but as you can use special
 # characters, you can't use some regular expressions and sub-expressions.
 ynh_replace_special_string () {
-	local match_string=$1
+	# Declare an array to define the options of this helper.
-	local replace_string=$2
+	local legacy_args=mrf
-	local workfile=$3
+	declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
+	local match_string
+	local replace_string
+	local target_file
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
 
-  # Escape any backslash to preserve them as simple backslash.
+	# Escape any backslash to preserve them as simple backslash.
-  match_string=${match_string//\\/"\\\\"}
+	match_string=${match_string//\\/"\\\\"}
-  replace_string=${replace_string//\\/"\\\\"}
+	replace_string=${replace_string//\\/"\\\\"}
 
 	# Escape the & character, who has a special function in sed.
 	match_string=${match_string//&/"\&"}
 	replace_string=${replace_string//&/"\&"}
 
-	ynh_replace_string "$match_string" "$replace_string" "$workfile"
+	ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file"
 }

data/helpers.d/system

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Manage a fail of the script
 #
 # [internal]
@@ -53,3 +55,107 @@ ynh_abort_if_errors () {
 ynh_get_debian_release () {
 	echo $(lsb_release --codename --short)
 }
+
+# Read the value of a key in a ynh manifest file
+#
+# usage: ynh_read_manifest manifest key
+# | arg: -m, --manifest= - Path of the manifest to read
+# | arg: -k, --key= - Name of the key to find
+ynh_read_manifest () {
+	# Declare an array to define the options of this helper.
+        declare -Ar args_array=( [m]=manifest= [k]=manifest_key= )
+        local manifest
+        local manifest_key
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	if [ ! -e "$manifest" ]; then
+		# If the manifest isn't found, try the common place for backup and restore script.
+		manifest="../settings/manifest.json"
+	fi
+
+	jq ".$manifest_key" "$manifest" --raw-output
+}
+
+# Read the upstream version from the manifest
+# The version number in the manifest is defined by <upstreamversion>~ynh<packageversion>
+# For example : 4.3-2~ynh3
+# This include the number before ~ynh
+# In the last example it return 4.3-2
+#
+# usage: ynh_app_upstream_version [-m manifest]
+# | arg: -m, --manifest= - Path of the manifest to read
+ynh_app_upstream_version () {
+    declare -Ar args_array=( [m]=manifest= )
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    manifest="${manifest:-../manifest.json}"
+    version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version")
+    echo "${version_key/~ynh*/}"
+}
+
+# Read package version from the manifest
+# The version number in the manifest is defined by <upstreamversion>~ynh<packageversion>
+# For example : 4.3-2~ynh3
+# This include the number after ~ynh
+# In the last example it return 3
+#
+# usage: ynh_app_package_version [-m manifest]
+# | arg: -m, --manifest= - Path of the manifest to read
+ynh_app_package_version () {
+    declare -Ar args_array=( [m]=manifest= )
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    manifest="${manifest:-../manifest.json}"
+    version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version")
+    echo "${version_key/*~ynh/}"
+}
+
+# Checks the app version to upgrade with the existing app version and returns:
+# - UPGRADE_APP if the upstream app version has changed
+# - UPGRADE_PACKAGE if only the YunoHost package has changed
+#
+## It stops the current script without error if the package is up-to-date
+#
+# This helper should be used to avoid an upgrade of an app, or the upstream part
+# of it, when it's not needed
+#
+# To force an upgrade, even if the package is up to date,
+# you have to set the variable YNH_FORCE_UPGRADE before.
+# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp
+#
+# usage: ynh_check_app_version_changed
+ynh_check_app_version_changed () {
+  local force_upgrade=${YNH_FORCE_UPGRADE:-0}
+  local package_check=${PACKAGE_CHECK_EXEC:-0}
+
+  # By default, upstream app version has changed
+  local return_value="UPGRADE_APP"
+
+  local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0)
+  local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")"
+  local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0)
+  local update_upstream_version="$(ynh_app_upstream_version)"
+
+  if [ "$current_version" == "$update_version" ] ; then
+      # Complete versions are the same
+      if [ "$force_upgrade" != "0" ]
+      then
+        echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2
+        unset YNH_FORCE_UPGRADE
+      elif [ "$package_check" != "0" ]
+      then
+        echo "Upgrade forced for package check." >&2
+      else
+        ynh_die "Up-to-date, nothing to do" 0
+      fi
+  elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then
+    # Upstream versions are the same, only YunoHost package versions differ
+    return_value="UPGRADE_PACKAGE"
+  fi
+  echo $return_value
+}

data/helpers.d/user

@@ -1,23 +1,40 @@
+#!/bin/bash
+
 # Check if a YunoHost user exists
 #
 # example: ynh_user_exists 'toto' || exit 1
 #
-# usage: ynh_user_exists username
+# usage: ynh_user_exists --username=username
-# | arg: username - the username to check
+# | arg: -u, --username - the username to check
 ynh_user_exists() {
-    sudo yunohost user list --output-as json | grep -q "\"username\": \"${1}\""
+    # Declare an array to define the options of this helper.
+    local legacy_args=u
+    declare -Ar args_array=( [u]=username= )
+    local username
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    sudo yunohost user list --output-as json | grep -q "\"username\": \"${username}\""
 }
 
 # Retrieve a YunoHost user information
 #
 # example: mail=$(ynh_user_get_info 'toto' 'mail')
 #
-# usage: ynh_user_get_info username key
+# usage: ynh_user_get_info --username=username --key=key
-# | arg: username - the username to retrieve info from
+# | arg: -u, --username - the username to retrieve info from
-# | arg: key - the key to retrieve
+# | arg: -k, --key - the key to retrieve
 # | ret: string - the key's value
 ynh_user_get_info() {
-    sudo yunohost user info "$1" --output-as plain | ynh_get_plain_key "$2"
+    # Declare an array to define the options of this helper.
+    local legacy_args=uk
+    declare -Ar args_array=( [u]=username= [k]=key= )
+    local username
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    sudo yunohost user info "$username" --output-as plain | ynh_get_plain_key "$key"
 }
 
 # Get the list of YunoHost users
@@ -33,10 +50,17 @@ ynh_user_list() {
 
 # Check if a user exists on the system
 #
-# usage: ynh_system_user_exists username
+# usage: ynh_system_user_exists --username=username
-# | arg: username - the username to check
+# | arg: -u, --username - the username to check
 ynh_system_user_exists() {
-    getent passwd "$1" &>/dev/null
+    # Declare an array to define the options of this helper.
+    local legacy_args=u
+    declare -Ar args_array=( [u]=username= )
+    local username
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    getent passwd "$username" &>/dev/null
 }
 
 # Create a system user
@@ -54,7 +78,7 @@ ynh_system_user_exists() {
 # If this argument is omitted, the user will be created with /usr/sbin/nologin shell
 ynh_system_user_create () {
 	# Declare an array to define the options of this helper.
-	local legacy_args=uh
+	local legacy_args=uhs
 	declare -Ar args_array=( [u]=username= [h]=home_dir= [s]=use_shell )
 	local username
 	local home_dir
@@ -82,14 +106,21 @@ ynh_system_user_create () {
 
 # Delete a system user
 #
-# usage: ynh_system_user_delete user_name
+# usage: ynh_system_user_delete --username=user_name
-# | arg: user_name - Name of the system user that will be create
+# | arg: -u, --username - Name of the system user that will be create
 ynh_system_user_delete () {
-    if ynh_system_user_exists "$1"	# Check if the user exists on the system
+    # Declare an array to define the options of this helper.
+    local legacy_args=u
+    declare -Ar args_array=( [u]=username= )
+    local username
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    if ynh_system_user_exists "$username"	# Check if the user exists on the system
     then
-		echo "Remove the user $1" >&2
+		echo "Remove the user $username" >&2
-		sudo userdel $1
+		sudo userdel $username
 	else
-		echo "The user $1 was not found" >&2
+		echo "The user $username was not found" >&2
     fi
 }

data/helpers.d/utils

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Extract a key from a plain command output
 #
 # example: yunohost user info tata --output-as plain | ynh_get_plain_key mail
@@ -49,7 +51,7 @@ ynh_restore_upgradebackup () {
 			sudo yunohost app remove $app
 			# Restore the backup
 			sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
-			ynh_die "The app was restored to the way it was before the failed upgrade."
+			ynh_die --message="The app was restored to the way it was before the failed upgrade."
 		fi
 	else
       		echo "\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" >&2
@@ -97,7 +99,7 @@ ynh_backup_before_upgrade () {
 				sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null
 			fi
 		else
-			ynh_die "Backup failed, the upgrade process was aborted."
+			ynh_die --message="Backup failed, the upgrade process was aborted."
 		fi
         else
                 echo "\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
@@ -118,6 +120,8 @@ ynh_backup_before_upgrade () {
 # SOURCE_FORMAT=tar.gz
 # # (Optional) Put false if sources are directly in the archive root
 # # default: true
+# # Instead of true, SOURCE_IN_SUBDIR could be the number of sub directories
+# # to remove.
 # SOURCE_IN_SUBDIR=false
 # # (Optionnal) Name of the local archive (offline setup support)
 # # default: ${src_id}.${src_format}
@@ -136,27 +140,35 @@ ynh_backup_before_upgrade () {
 # If it's ok, the source archive will be uncompressed in $dest_dir. If the
 # SOURCE_IN_SUBDIR is true, the first level directory of the archive will be
 # removed.
+# If SOURCE_IN_SUBDIR is a numeric value, 2 for example, the 2 first level
+# directories will be removed
 #
 # Finally, patches named sources/patches/${src_id}-*.patch and extra files in
 # sources/extra_files/$src_id will be applied to dest_dir
 #
 #
-# usage: ynh_setup_source dest_dir [source_id]
+# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id]
-# | arg: dest_dir  - Directory where to setup sources
+# | arg: -d, --dest_dir  - Directory where to setup sources
-# | arg: source_id - Name of the app, if the package contains more than one app
+# | arg: -s, --source_id - Name of the app, if the package contains more than one app
 ynh_setup_source () {
-    local dest_dir=$1
+    # Declare an array to define the options of this helper.
-    local src_id=${2:-app} # If the argument is not given, source_id equals "app"
+    local legacy_args=ds
+    declare -Ar args_array=( [d]=dest_dir= [s]=source_id= )
+    local dest_dir
+    local source_id
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    source_id="${source_id:-app}" # If the argument is not given, source_id equals "app"
 
     # Load value from configuration file (see above for a small doc about this file
     # format)
-    local src_url=$(grep 'SOURCE_URL=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-)
+    local src_url=$(grep 'SOURCE_URL=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
-    local src_sum=$(grep 'SOURCE_SUM=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-)
+    local src_sum=$(grep 'SOURCE_SUM=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
-    local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-)
+    local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
-    local src_format=$(grep 'SOURCE_FORMAT=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-)
+    local src_format=$(grep 'SOURCE_FORMAT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
-    local src_extract=$(grep 'SOURCE_EXTRACT=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-)
+    local src_extract=$(grep 'SOURCE_EXTRACT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
-    local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-)
+    local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
-    local src_filename=$(grep 'SOURCE_FILENAME=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-)
+    local src_filename=$(grep 'SOURCE_FILENAME=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
 
     # Default value
     src_sumprg=${src_sumprg:-sha256sum}
@@ -165,7 +177,7 @@ ynh_setup_source () {
     src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]')
     src_extract=${src_extract:-true}
     if [ "$src_filename" = "" ] ; then
-        src_filename="${src_id}.${src_format}"
+        src_filename="${source_id}.${src_format}"
     fi
     local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}"
 
@@ -173,16 +185,16 @@ ynh_setup_source () {
     then    # Use the local source file if it is present
         cp $local_src $src_filename
     else    # If not, download the source
-        local out=`wget -nv -O $src_filename $src_url 2>&1` || ynh_print_err $out
+        local out=`wget -nv -O $src_filename $src_url 2>&1` || ynh_print_err --message="$out"
     fi
 
     # Check the control sum
     echo "${src_sum} ${src_filename}" | ${src_sumprg} -c --status \
-        || ynh_die "Corrupt source"
+        || ynh_die --message="Corrupt source"
 
     # Extract source into the app dir
     mkdir -p "$dest_dir"
-    
+
     if ! "$src_extract"
     then
         mv $src_filename $dest_dir
@@ -194,35 +206,41 @@ ynh_setup_source () {
             local tmp_dir=$(mktemp -d)
             unzip -quo $src_filename -d "$tmp_dir"
             cp -a $tmp_dir/*/. "$dest_dir"
-            ynh_secure_remove "$tmp_dir"
+            ynh_secure_remove --file="$tmp_dir"
         else
             unzip -quo $src_filename -d "$dest_dir"
         fi
     else
         local strip=""
-        if $src_in_subdir ; then
+        if [ "$src_in_subdir" != "false" ]
-            strip="--strip-components 1"
+        then
+            if [ "$src_in_subdir" == "true" ]; then
+                local sub_dirs=1
+            else
+                local sub_dirs="$src_in_subdir"
+            fi
+            strip="--strip-components $sub_dirs"
         fi
         if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]] ; then
             tar -xf $src_filename -C "$dest_dir" $strip
         else
-            ynh_die "Archive format unrecognized."
+            ynh_die --message="Archive format unrecognized."
         fi
     fi
 
     # Apply patches
-    if (( $(find $YNH_CWD/../sources/patches/ -type f -name "${src_id}-*.patch" 2> /dev/null | wc -l) > "0" )); then
+    if (( $(find $YNH_CWD/../sources/patches/ -type f -name "${source_id}-*.patch" 2> /dev/null | wc -l) > "0" )); then
         local old_dir=$(pwd)
         (cd "$dest_dir" \
-            && for p in $YNH_CWD/../sources/patches/${src_id}-*.patch; do \
+            && for p in $YNH_CWD/../sources/patches/${source_id}-*.patch; do \
                 patch -p1 < $p; done) \
-            || ynh_die "Unable to apply patches"
+            || ynh_die --message="Unable to apply patches"
         cd $old_dir
     fi
 
     # Add supplementary files
-    if test -e "$YNH_CWD/../sources/extra_files/${src_id}"; then
+    if test -e "$YNH_CWD/../sources/extra_files/${source_id}"; then
-        cp -a $YNH_CWD/../sources/extra_files/$src_id/. "$dest_dir"
+        cp -a $YNH_CWD/../sources/extra_files/$source_id/. "$dest_dir"
     fi
 }
 
@@ -239,7 +257,14 @@ ynh_setup_source () {
 # | arg: ...         - (Optionnal) More POST keys and values
 ynh_local_curl () {
 	# Define url of page to curl
-	local full_page_url=https://localhost$path_url$1
+	local local_page=$(ynh_normalize_url_path $1)
+	local full_path=$path_url$local_page
+	
+	if [ "${path_url}" == "/" ]; then 
+		full_path=$local_page
+	fi
+	
+	local full_page_url=https://localhost$full_path
 
 	# Concatenate all other arguments with '&' to prepare POST data
 	local POST_data=""

data/hooks/backup/05-conf_ldap

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ldap"

data/hooks/backup/08-conf_ssh

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ssh"

data/hooks/backup/11-conf_ynh_mysql

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh/mysql"

data/hooks/backup/14-conf_ssowat

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ssowat"

data/hooks/backup/17-data_home

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/data/home"

data/hooks/backup/20-conf_ynh_firewall

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh/firewall"

data/hooks/backup/21-conf_ynh_certs

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh/certs"

data/hooks/backup/23-data_mail

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/data/mail"

data/hooks/backup/26-conf_xmpp

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/xmpp"

data/hooks/backup/29-conf_nginx

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/nginx"

data/hooks/backup/32-conf_cron

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/cron"

data/hooks/backup/40-conf_ynh_currenthost

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh"

data/hooks/conf_regen/03-ssh

@@ -2,7 +2,7 @@
 
 set -e
 
-. /usr/share/yunohost/helpers.d/utils
+. /usr/share/yunohost/helpers
 
 do_pre_regen() {
     pending_dir=$1

data/hooks/conf_regen/15-nginx

@@ -2,7 +2,7 @@
 
 set -e
 
-. /usr/share/yunohost/helpers.d/utils
+. /usr/share/yunohost/helpers
 
 do_init_regen() {
   if [[ $EUID -ne 0 ]]; then

data/hooks/conf_regen/34-mysql

@@ -2,6 +2,7 @@
 
 set -e
 MYSQL_PKG="mariadb-server-10.1"
+. /usr/share/yunohost/helpers
 
 do_pre_regen() {
   pending_dir=$1
@@ -15,7 +16,6 @@ do_post_regen() {
   regen_conf_files=$1
 
   if [ ! -f /etc/yunohost/mysql ]; then
-      . /usr/share/yunohost/helpers.d/string
 
       # ensure that mysql is running
       sudo systemctl -q is-active mysql.service \
@@ -25,8 +25,6 @@ do_post_regen() {
       mysql_password=$(ynh_string_random 10)
       sudo mysqladmin -s -u root -pyunohost password "$mysql_password" || {
         if [ $FORCE -eq 1 ]; then
-            . /usr/share/yunohost/helpers.d/package
-
             echo "It seems that you have already configured MySQL." \
               "YunoHost needs to have a root access to MySQL to runs its" \
               "applications, and is going to reset the MySQL root password." \

data/hooks/conf_regen/43-dnsmasq

@@ -1,13 +1,11 @@
 #!/bin/bash
 
 set -e
+. /usr/share/yunohost/helpers
 
 do_pre_regen() {
   pending_dir=$1
 
-  # source ip helpers
-  . /usr/share/yunohost/helpers.d/ip
-
   cd /usr/share/yunohost/templates/dnsmasq
 
   # create directory for pending conf

data/hooks/restore/11-conf_ynh_mysql

@@ -1,6 +1,8 @@
 backup_dir="$1/conf/ynh/mysql"
 MYSQL_PKG="mariadb-server-10.1"
 
+. /usr/share/yunohost/helpers
+
 # ensure that mysql is running
 service mysql status >/dev/null 2>&1 \
   || service mysql start
@@ -11,13 +13,11 @@ service mysql status >/dev/null 2>&1 \
 new_pwd=$(sudo cat "${backup_dir}/root_pwd" || sudo cat "${backup_dir}/mysql")
 [ -z "$curr_pwd" ] && curr_pwd="yunohost"
 [ -z "$new_pwd" ] && {
-    . /usr/share/yunohost/helpers.d/string
     new_pwd=$(ynh_string_random 10)
 }
 
 # attempt to change it
 sudo mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || {
-  . /usr/share/yunohost/helpers.d/package
 
   echo "It seems that you have already configured MySQL." \
     "YunoHost needs to have a root access to MySQL to runs its" \

data/templates/dnsmasq/plain/resolv.dnsmasq.conf

@@ -14,15 +14,17 @@ nameserver 80.67.169.40
 nameserver 80.67.188.188
 # (FR) ARN
 nameserver 89.234.141.66
+# (FR) Aquilenet
+nameserver 185.233.100.100
+nameserver 185.233.100.101
 # (FR) gozmail / grifon
-nameserver 89.234.186.18
+nameserver 80.67.190.200
 # (DE) FoeBud / Digital Courage
 nameserver 85.214.20.141
-# (FR) Aquilenet [added manually, following comments from @sachaz]
-nameserver 141.255.128.100
-nameserver 141.255.128.101
 # (DE) CCC Berlin
-nameserver 213.73.91.35
+nameserver 195.160.173.53
+# (DE) AS250
+nameserver 194.150.168.168
 # (DE) Ideal-Hosting
 nameserver 84.200.69.80
 nameserver 84.200.70.40

data/templates/fail2ban/jail.conf

@@ -513,27 +513,27 @@ logpath  = %(vsftpd_log)s
 # ASSP SMTP Proxy Jail
 [assp]
 
-port     = smtp,465,submission
+port     = smtp,submission
 logpath  = /root/path/to/assp/logs/maillog.txt
 
 
 [courier-smtp]
 
-port     = smtp,465,submission
+port     = smtp,submission
 logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
 
 
 [postfix]
 
-port     = smtp,465,submission
+port     = smtp,submission
 logpath  = %(postfix_log)s
 backend  = %(postfix_backend)s
 
 
 [postfix-rbl]
 
-port     = smtp,465,submission
+port     = smtp,submission
 logpath  = %(postfix_log)s
 backend  = %(postfix_backend)s
 maxretry = 1
@@ -541,14 +541,14 @@ maxretry = 1
 
 [sendmail-auth]
 
-port    = submission,465,smtp
+port    = submission,smtp
 logpath = %(syslog_mail)s
 backend = %(syslog_backend)s
 
 
 [sendmail-reject]
 
-port     = smtp,465,submission
+port     = smtp,submission
 logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
 
@@ -556,7 +556,7 @@ backend  = %(syslog_backend)s
 [qmail-rbl]
 
 filter  = qmail
-port    = smtp,465,submission
+port    = smtp,submission
 logpath = /service/qmail/log/main/current
 
 
@@ -564,14 +564,14 @@ logpath = /service/qmail/log/main/current
 # but can be set by syslog_facility in the dovecot configuration.
 [dovecot]
 
-port    = pop3,pop3s,imap,imaps,submission,465,sieve
+port    = pop3,pop3s,imap,imaps,submission,sieve
 logpath = %(dovecot_log)s
 backend = %(dovecot_backend)s
 
 
 [sieve]
 
-port   = smtp,465,submission
+port   = smtp,submission
 logpath = %(dovecot_log)s
 backend = %(dovecot_backend)s
 
@@ -584,19 +584,19 @@ logpath = %(solidpop3d_log)s
 
 [exim]
 
-port   = smtp,465,submission
+port   = smtp,submission
 logpath = %(exim_main_log)s
 
 
 [exim-spam]
 
-port   = smtp,465,submission
+port   = smtp,submission
 logpath = %(exim_main_log)s
 
 
 [kerio]
 
-port    = imap,smtp,imaps,465
+port    = imap,smtp,imaps
 logpath = /opt/kerio/mailserver/store/logs/security.log
 
 
@@ -607,14 +607,14 @@ logpath = /opt/kerio/mailserver/store/logs/security.log
 
 [courier-auth]
 
-port     = smtp,465,submission,imaps,pop3,pop3s
+port     = smtp,submission,imaps,pop3,pop3s
 logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
 
 
 [postfix-sasl]
 
-port     = smtp,465,submission,imap,imaps,pop3,pop3s
+port     = smtp,submission,imap,imaps,pop3,pop3s
 # You might consider monitoring /var/log/mail.warn instead if you are
 # running postfix since it would provide the same log lines at the
 # "warn" level but overall at the smaller filesize.
@@ -631,7 +631,7 @@ backend = %(syslog_backend)s
 
 [squirrelmail]
 
-port = smtp,465,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks
+port = smtp,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks
 logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log
 
 

data/templates/nginx/plain/yunohost_admin.conf

@@ -63,7 +63,8 @@ server {
         if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) {
             return 403;
         }
-
+        # X-Robots-Tag to precise the rules applied.
+        add_header  X-Robots-Tag "nofollow, noindex, noarchive, nosnippet";
         # Redirect most of 404 to maindomain.tld/yunohost/sso
         access_by_lua_file /usr/share/ssowat/access.lua;
     }

data/templates/nginx/server.tpl.conf

@@ -53,8 +53,10 @@ server {
 
     # Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners
     # https://wiki.mozilla.org/Security/Guidelines/Web_Security
-    # https://observatory.mozilla.org/
+    # https://observatory.mozilla.org/ 
+    {% if domain_cert_ca != "Self-signed" %}
     more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload";
+    {% endif %}
     more_set_headers "Content-Security-Policy : upgrade-insecure-requests";
     more_set_headers "Content-Security-Policy-Report-Only : default-src https: data: 'unsafe-inline' 'unsafe-eval'";
     more_set_headers "X-Content-Type-Options : nosniff";

debian/changelog

@@ -1,3 +1,16 @@
+yunohost (3.4.2.4) stable; urgency=low
+
+  - [fix] Meltdown vulnerability checker something outputing trash instead of pure json
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Tue, 19 Feb 2019 19:11:38  +0000
+
+yunohost (3.4.2.3) stable; urgency=low
+
+  - [fix] Admin password appearing in logs after logging in on webadmin
+  - [fix] Update friendly DNS resolver list
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Thu, 07 Feb 2019 03:20:10 +0000
+
 yunohost (3.4.2.2) stable; urgency=low
 
   - Silly bug in migraton 8 :|

debian/control

@@ -13,7 +13,7 @@ Depends: ${python:Depends}, ${misc:Depends}
  , moulinette (>= 2.7.1), ssowat (>= 2.7.1)
  , python-psutil, python-requests, python-dnspython, python-openssl
  , python-apt, python-miniupnpc, python-dbus, python-jinja2
- , glances
+ , glances, apt-transport-https
  , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq
  , ca-certificates, netcat-openbsd, iproute
  , mariadb-server, php-mysql | php-mysqlnd

locales/en.json

@@ -24,7 +24,8 @@
     "app_location_install_failed": "Unable to install the app in this location because it conflit with the app '{other_app}' already installed on '{other_path}'",
     "app_location_unavailable": "This url is not available or conflicts with the already installed app(s):\n{apps:s}",
     "app_manifest_invalid": "Invalid app manifest: {error}",
-    "app_no_upgrade": "No app to upgrade",
+    "app_no_upgrade": "No apps to upgrade",
+    "app_not_upgraded": "The following apps were not upgraded: {apps}",
     "app_not_correctly_installed": "{app:s} seems to be incorrectly installed",
     "app_not_installed": "{app:s} is not installed",
     "app_not_properly_removed": "{app:s} has not been properly removed",
@@ -34,9 +35,14 @@
     "app_requirements_failed": "Unable to meet requirements for {app}: {error}",
     "app_requirements_unmeet": "Requirements are not met for {app}, the package {pkgname} ({version}) must be {spec}",
     "app_sources_fetch_failed": "Unable to fetch sources files",
+    "app_start_install": "Installing application {app}…",
+    "app_start_remove": "Removing application {app}…",
+    "app_start_backup": "Collecting files to be backuped for {app}…",
+    "app_start_restore": "Restoring application {app}…",
     "app_unknown": "Unknown app",
     "app_unsupported_remote_type": "Unsupported remote type used for the app",
-    "app_upgrade_app_name": "Upgrading app {app}…",
+    "app_upgrade_several_apps": "The following apps will be upgraded : {apps}",
+    "app_upgrade_app_name": "Now upgrading app {app}…",
     "app_upgrade_failed": "Unable to upgrade {app:s}",
     "app_upgrade_some_app_failed": "Unable to upgrade some applications",
     "app_upgraded": "{app:s} has been upgraded",
@@ -63,6 +69,7 @@
     "ask_path": "Path",
     "backup_abstract_method": "This backup method hasn't yet been implemented",
     "backup_action_required": "You must specify something to save",
+    "backup_actually_backuping": "Now creating a backup archive from the files collected…",
     "backup_app_failed": "Unable to back up the app '{app:s}'",
     "backup_applying_method_borg": "Sending all files to backup into borg-backup repository…",
     "backup_applying_method_copy": "Copying all files to backup…",
@@ -75,7 +82,7 @@
     "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'",
     "backup_archive_open_failed": "Unable to open the backup archive",
     "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup",
-    "backup_archive_writing_error": "Unable to add files to backup into the compressed archive",
+    "backup_archive_writing_error": "Unable to add files '{source:s}' (named in the archive: '{dest:s}') to backup into the compressed archive '{archive:s}'",
     "backup_ask_for_copying_if_needed": "Some files couldn't be prepared to be backuped using the method that avoid to temporarily waste space on the system. To perform the backup, {size:s}MB should be used temporarily. Do you agree?",
     "backup_borg_not_implemented": "Borg backup method is not yet implemented",
     "backup_cant_mount_uncompress_archive": "Unable to mount in readonly mode the uncompress archive directory",
@@ -99,6 +106,7 @@
     "backup_method_copy_finished": "Backup copy finished",
     "backup_method_custom_finished": "Custom backup method '{method:s}' finished",
     "backup_method_tar_finished": "Backup tar archive created",
+    "backup_mount_archive_for_restore": "Preparing archive for restoration…",
     "backup_no_uncompress_archive_dir": "Uncompress archive directory doesn't exist",
     "backup_nothings_done": "There is nothing to save",
     "backup_output_directory_forbidden": "Forbidden output directory. Backups can't be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders",
@@ -106,7 +114,6 @@
     "backup_output_directory_required": "You must provide an output directory for the backup",
     "backup_output_symlink_dir_broken": "You have a broken symlink instead of your archives directory '{path:s}'. You may have a specific setup to backup your data on an other filesystem, in this case you probably forgot to remount or plug your hard dirve or usb key.",
     "backup_php5_to_php7_migration_may_fail": "Could not convert your archive to support php7, your php apps may fail to restore (reason: {error:s})",
-    "backup_running_app_script": "Running backup script of app '{app:s}'…",
     "backup_running_hooks": "Running backup hooks…",
     "backup_system_part_failed": "Unable to backup the '{part:s}' system part",
     "backup_unable_to_organize_files": "Unable to organize files in the archive with the quick method",
@@ -146,6 +153,7 @@
     "diagnosis_monitor_network_error": "Can't monitor network: {error}",
     "diagnosis_monitor_system_error": "Can't monitor system: {error}",
     "diagnosis_no_apps": "No installed application",
+    "dpkg_is_broken": "You cannot do this right now because dpkg/apt (the system package managers) seems to be in a broken state... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.",
     "dnsmasq_isnt_installed": "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'",
     "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first",
     "domain_cert_gen_failed": "Unable to generate certificate",
@@ -465,6 +473,7 @@
     "ssowat_persistent_conf_write_error": "Error while saving SSOwat persistent configuration: {error:s}. Edit /etc/ssowat/conf.json.persistent file to fix the JSON syntax",
     "system_upgraded": "The system has been upgraded",
     "system_username_exists": "Username already exists in the system users",
+    "this_action_broke_dpkg": "This action broke dpkg/apt (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.",
     "unbackup_app": "App '{app:s}' will not be saved",
     "unexpected_error": "An unexpected error occured: {error}",
     "unit_unknown": "Unknown unit '{unit:s}'",

src/yunohost/app.py

@@ -97,6 +97,9 @@ def app_fetchlist(url=None, name=None):
         name -- Name of the list
         url -- URL of remote JSON list
     """
+    if not url.endswith(".json"):
+        raise YunohostError("This is not a valid application list url. It should end with .json.")
+
     # If needed, create folder where actual appslists are stored
     if not os.path.exists(REPO_PATH):
         os.makedirs(REPO_PATH)
@@ -564,6 +567,9 @@ def app_upgrade(auth, app=[], url=None, file=None):
         url -- Git url to fetch for upgrade
 
     """
+    if packages.dpkg_is_broken():
+        raise YunohostError("dpkg_is_broken")
+
     from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
 
     # Retrieve interface
@@ -574,7 +580,7 @@ def app_upgrade(auth, app=[], url=None, file=None):
     except YunohostError:
         raise YunohostError('app_no_upgrade')
 
-    upgraded_apps = []
+    not_upgraded_apps = []
 
     apps = app
     user_specified_list = True
@@ -586,7 +592,13 @@ def app_upgrade(auth, app=[], url=None, file=None):
     elif not isinstance(app, list):
         apps = [app]
 
-    logger.info("Upgrading apps %s", ", ".join(app))
+    # Remove possible duplicates
+    apps = [app for i,app in enumerate(apps) if apps not in L[:i]]
+
+    if len(apps) == 0:
+        raise YunohostError('app_no_upgrade')
+    if len(apps) > 1:
+        logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(app)))
 
     for app_instance_name in apps:
         logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name))
@@ -594,9 +606,6 @@ def app_upgrade(auth, app=[], url=None, file=None):
         if not installed:
             raise YunohostError('app_not_installed', app=app_instance_name)
 
-        if app_instance_name in upgraded_apps:
-            continue
-
         app_dict = app_info(app_instance_name, raw=True)
 
         if file:
@@ -648,6 +657,7 @@ def app_upgrade(auth, app=[], url=None, file=None):
         if hook_exec(extracted_app_folder + '/scripts/upgrade',
                      args=args_list, env=env_dict) != 0:
             msg = m18n.n('app_upgrade_failed', app=app_instance_name)
+            not_upgraded_apps.append(app_instance_name)
             logger.error(msg)
             operation_logger.error(msg)
         else:
@@ -675,14 +685,13 @@ def app_upgrade(auth, app=[], url=None, file=None):
                     os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path))
 
             # So much win
-            upgraded_apps.append(app_instance_name)
             logger.success(m18n.n('app_upgraded', app=app_instance_name))
 
             hook_callback('post_app_upgrade', args=args_list, env=env_dict)
             operation_logger.success()
 
-    if not upgraded_apps:
+    if not_upgraded_apps:
-        raise YunohostError('app_no_upgrade')
+        raise YunohostError('app_not_upgraded', apps=', '.join(not_upgraded_apps))
 
     app_ssowatconf(auth)
 
@@ -705,6 +714,9 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on
         no_remove_on_failure -- Debug option to avoid removing the app on a failed installation
         force -- Do not ask for confirmation when installing experimental / low-quality apps
     """
+    if packages.dpkg_is_broken():
+        raise YunohostError("dpkg_is_broken")
+
     from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
     from yunohost.log import OperationLogger
 
@@ -734,8 +746,8 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on
         if answer.upper() != "Y":
             raise YunohostError("aborting")
 
-
     raw_app_list = app_list(raw=True)
+
     if app in raw_app_list or ('@' in app) or ('http://' in app) or ('https://' in app):
         if app in raw_app_list:
             state = raw_app_list[app].get("state", "notworking")
@@ -798,6 +810,8 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on
     operation_logger.related_to.append(("app", app_id))
     operation_logger.start()
 
+    logger.info(m18n.n("app_start_install", app=app_id))
+
     # Create app directory
     app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
     if os.path.exists(app_setting_path):
@@ -874,6 +888,9 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on
 
             app_ssowatconf(auth)
 
+            if packages.dpkg_is_broken():
+                logger.error(m18n.n("this_action_broke_dpkg"))
+
             if install_retcode == -1:
                 msg = m18n.n('operation_interrupted') + " " + error_msg
                 raise YunohostError(msg, raw_msg=True)
@@ -918,6 +935,8 @@ def app_remove(operation_logger, auth, app):
 
     operation_logger.start()
 
+    logger.info(m18n.n("app_start_remove", app=app))
+
     app_setting_path = APPS_SETTING_PATH + app
 
     # TODO: display fail messages from script
@@ -956,6 +975,9 @@ def app_remove(operation_logger, auth, app):
     hook_remove(app)
     app_ssowatconf(auth)
 
+    if packages.dpkg_is_broken():
+        raise YunohostError("this_action_broke_dpkg")
+
 
 def app_addaccess(auth, apps, users=[]):
     """
@@ -1173,7 +1195,7 @@ def app_makedefault(operation_logger, auth, app, domain=None):
         with open('/etc/ssowat/conf.json.persistent') as json_conf:
             ssowat_conf = json.loads(str(json_conf.read()))
     except ValueError as e:
-        raise YunohostError('ssowat_persistent_conf_read_error', error=e.strerror)
+        raise YunohostError('ssowat_persistent_conf_read_error', error=e)
     except IOError:
         ssowat_conf = {}
 
@@ -1186,7 +1208,7 @@ def app_makedefault(operation_logger, auth, app, domain=None):
         with open('/etc/ssowat/conf.json.persistent', 'w+') as f:
             json.dump(ssowat_conf, f, sort_keys=True, indent=4)
     except IOError as e:
-        raise YunohostError('ssowat_persistent_conf_write_error', error=e.strerror)
+        raise YunohostError('ssowat_persistent_conf_write_error', error=e)
 
     os.system('chmod 644 /etc/ssowat/conf.json.persistent')
 
@@ -1209,8 +1231,8 @@ def app_setting(app, key, value=None, delete=False):
     if value is None and not delete:
         try:
             return app_settings[key]
-        except:
+        except Exception as e:
-            logger.debug("cannot get app setting '%s' for '%s'", key, app)
+            logger.debug("cannot get app setting '%s' for '%s' (%s)", key, app, e)
             return None
     else:
         if delete and key in app_settings:
@@ -1394,7 +1416,8 @@ def app_ssowatconf(auth):
 
     try:
         apps_list = app_list(installed=True)['apps']
-    except:
+    except Exception as e:
+        logger.debug("cannot get installed app list because %s", e)
         apps_list = []
 
     def _get_setting(settings, name):
@@ -1824,7 +1847,7 @@ def _extract_app_from_file(path, remove=False):
     except IOError:
         raise YunohostError('app_install_files_invalid')
     except ValueError as e:
-        raise YunohostError('app_manifest_invalid', error=e.strerror)
+        raise YunohostError('app_manifest_invalid', error=e)
 
     logger.debug(m18n.n('done'))
 
@@ -1907,7 +1930,7 @@ def _fetch_app_from_git(app):
                 # we will be able to use it. Without this option all the history
                 # of the submodules repo is downloaded.
                 subprocess.check_call([
-                    'git', 'clone', '-b',  branch, '--single-branch', '--recursive', '--depth=1', url,
+                    'git', 'clone', '-b', branch, '--single-branch', '--recursive', '--depth=1', url,
                     extracted_app_folder])
                 subprocess.check_call([
                     'git', 'reset', '--hard', branch
@@ -1917,7 +1940,7 @@ def _fetch_app_from_git(app):
             except subprocess.CalledProcessError:
                 raise YunohostError('app_sources_fetch_failed')
             except ValueError as e:
-                raise YunohostError('app_manifest_invalid', error=e.strerror)
+                raise YunohostError('app_manifest_invalid', error=e)
             else:
                 logger.debug(m18n.n('done'))
 
@@ -1925,8 +1948,8 @@ def _fetch_app_from_git(app):
         manifest['remote'] = {'type': 'git', 'url': url, 'branch': branch}
         try:
             revision = _get_git_last_commit_hash(url, branch)
-        except:
+        except Exception as e:
-            pass
+            logger.debug("cannot get last commit hash because: %s ", e)
         else:
             manifest['remote']['revision'] = revision
     else:
@@ -1970,7 +1993,7 @@ def _fetch_app_from_git(app):
             except subprocess.CalledProcessError:
                 raise YunohostError('app_sources_fetch_failed')
             except ValueError as e:
-                raise YunohostError('app_manifest_invalid', error=e.strerror)
+                raise YunohostError('app_manifest_invalid', error=e)
             else:
                 logger.debug(m18n.n('done'))
 
@@ -2250,7 +2273,7 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None):
             try:
                 user_info(auth, arg_value)
             except YunohostError as e:
-                raise YunohostError('app_argument_invalid', name=arg_name, error=e.strerror)
+                raise YunohostError('app_argument_invalid', name=arg_name, error=e)
         elif arg_type == 'app':
             if not _is_installed(arg_value):
                 raise YunohostError('app_argument_invalid', name=arg_name, error=m18n.n('app_unknown'))
@@ -2344,6 +2367,7 @@ def _parse_app_instance_name(app_instance_name):
     True
     """
     match = re_app_instance_name.match(app_instance_name)
+    assert match, "Could not parse app instance name : %s" % app_instance_name
     appid = match.groupdict().get('appid')
     app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1
     return (appid, app_instance_nb)

src/yunohost/backup.py

@@ -668,7 +668,7 @@ class BackupManager():
         tmp_app_bkp_dir = env_dict["YNH_APP_BACKUP_DIR"]
         settings_dir = os.path.join(self.work_dir, 'apps', app, 'settings')
 
-        logger.debug(m18n.n('backup_running_app_script', app=app))
+        logger.info(m18n.n("app_start_backup", app=app))
         try:
             # Prepare backup directory for the app
             filesystem.mkdir(tmp_app_bkp_dir, 0o750, True, uid='admin')
@@ -1242,6 +1242,8 @@ class RestoreManager():
         operation_logger = OperationLogger('backup_restore_app', related_to)
         operation_logger.start()
 
+        logger.info(m18n.n("app_start_restore", app=app_instance_name))
+
         # Check if the app is not already installed
         if _is_installed(app_instance_name):
             logger.error(m18n.n('restore_already_installed_app',
@@ -1802,10 +1804,11 @@ class TarBackupMethod(BackupMethod):
                 # Add the "source" into the archive and transform the path into
                 # "dest"
                 tar.add(path['source'], arcname=path['dest'])
-            tar.close()
         except IOError:
-            logger.error(m18n.n('backup_archive_writing_error'), exc_info=1)
+            logger.error(m18n.n('backup_archive_writing_error', source=path['source'], archive=self._archive_file, dest=path['dest']), exc_info=1)
             raise YunohostError('backup_creation_failed')
+        finally:
+            tar.close()
 
         # Move info file
         shutil.copy(os.path.join(self.work_dir, 'info.json'),
@@ -2059,6 +2062,7 @@ def backup_create(name=None, description=None, methods=[],
     backup_manager.collect_files()
 
     # Apply backup methods on prepared files
+    logger.info(m18n.n("backup_actually_backuping"))
     backup_manager.backup()
 
     logger.success(m18n.n('backup_created'))
@@ -2127,6 +2131,7 @@ def backup_restore(auth, name, system=[], apps=[], force=False):
     # Mount the archive then call the restore for each system part / app    #
     #
 
+    logger.info(m18n.n("backup_mount_archive_for_restore"))
     restore_manager.mount()
     restore_manager.restore()
 

src/yunohost/firewall.py

@@ -195,6 +195,7 @@ def firewall_reload(skip_upnp=False):
 
     """
     from yunohost.hook import hook_callback
+    from yunohost.service import _run_service_command
 
     reloaded = False
     errors = False
@@ -276,8 +277,7 @@ def firewall_reload(skip_upnp=False):
         # Refresh port forwarding with UPnP
         firewall_upnp(no_refresh=False)
 
-    # TODO: Use service_restart
+    _run_service_command("reload", "fail2ban")
-    os.system("service fail2ban restart")
 
     if errors:
         logger.warning(m18n.n('firewall_rules_cmd_failed'))

src/yunohost/settings.py

@@ -120,10 +120,18 @@ def settings_set(key, value):
         raise YunohostError('global_settings_unknown_type', setting=key,
                             unknown_type=key_type)
 
+    old_value = settings[key].get("value")
     settings[key]["value"] = value
-
     _save_settings(settings)
 
+    # TODO : whatdo if the old value is the same as
+    # the new value...
+    try:
+        trigger_post_change_hook(key, old_value, value)
+    except Exception as e:
+        logger.error("Post-change hook for setting %s failed : %s" % (key, e))
+        raise
+
 
 def settings_reset(key):
     """
@@ -241,6 +249,40 @@ def _save_settings(settings, location=SETTINGS_PATH):
     except Exception as e:
         raise YunohostError('global_settings_cant_write_settings', reason=e)
 
+
+# Meant to be a dict of setting_name -> function to call
+post_change_hooks = {}
+
+
+def post_change_hook(setting_name):
+    def decorator(func):
+        assert setting_name in DEFAULTS.keys(), "The setting %s does not exists" % setting_name
+        assert setting_name not in post_change_hooks, "You can only register one post change hook per setting (in particular for %s)" % setting_name
+        post_change_hooks[setting_name] = func
+        return func
+    return decorator
+
+
+def trigger_post_change_hook(setting_name, old_value, new_value):
+    if setting_name not in post_change_hooks:
+        logger.debug("Nothing to do after changing setting %s" % setting_name)
+        return
+
+    f = post_change_hooks[setting_name]
+    f(setting_name, old_value, new_value)
+
+
+# ===========================================
+#
+# Actions to trigger when changing a setting
+# You can define such an action with :
+#
+# @post_change_hook("your.setting.name")
+# def some_function_name(setting_name, old_value, new_value):
+#     # Do some stuff
+#
+# ===========================================
+
 @post_change_hook("security.ciphers.compatibility")
 def reconfigure_nginx(setting_name, old_value, new_value):
     if old_value != new_value:
@@ -250,4 +292,3 @@ def reconfigure_nginx(setting_name, old_value, new_value):
 def reconfigure_ssh(setting_name, old_value, new_value):
     if old_value != new_value:
         service_regen_conf(names=['ssh'], force=True)
-

src/yunohost/tools.py

@@ -524,6 +524,10 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal
         ignore_packages -- Ignore APT packages upgrade
 
     """
+    from yunohost.utils import packages
+    if packages.dpkg_is_broken():
+        raise YunohostError("dpkg_is_broken")
+
     failure = False
 
     # Retrieve interface
@@ -713,6 +717,22 @@ def tools_diagnosis(auth, private=False):
 def _check_if_vulnerable_to_meltdown():
     # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754
 
+    # We use a cache file to avoid re-running the script so many times,
+    # which can be expensive (up to around 5 seconds on ARM)
+    # and make the admin appear to be slow (c.f. the calls to diagnosis
+    # from the webadmin)
+    #
+    # The cache is in /tmp and shall disappear upon reboot
+    # *or* we compare it to dpkg.log modification time
+    # such that it's re-ran if there was package upgrades
+    # (e.g. from yunohost)
+    cache_file = "/tmp/yunohost-meltdown-diagnosis"
+    dpkg_log = "/var/log/dpkg.log"
+    if os.path.exists(cache_file):
+        if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log):
+            logger.debug("Using cached results for meltdown checker, from %s" % cache_file)
+            return read_json(cache_file)[0]["VULNERABLE"]
+
     # script taken from https://github.com/speed47/spectre-meltdown-checker
     # script commit id is store directly in the script
     file_dir = os.path.split(__file__)[0]
@@ -722,6 +742,7 @@ def _check_if_vulnerable_to_meltdown():
     # example output from the script:
     # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}]
     try:
+        logger.debug("Running meltdown vulnerability checker")
         call = subprocess.Popen("bash %s --batch json --variant 3" %
                                 SCRIPT_PATH, shell=True,
                                 stdout=subprocess.PIPE,
@@ -735,6 +756,14 @@ def _check_if_vulnerable_to_meltdown():
         output, err = call.communicate()
         assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
 
+        # If there are multiple lines, sounds like there was some messages
+        # in stdout that are not json >.> ... Try to get the actual json
+        # stuff which should be the last line
+        output = output.strip()
+        if "\n" in output:
+            logger.debug("Original meltdown checker output : %s" % output)
+            output = output.split("\n")[-1]
+
         CVEs = json.loads(output)
         assert len(CVEs) == 1
         assert CVEs[0]["NAME"] == "MELTDOWN"
@@ -744,6 +773,8 @@ def _check_if_vulnerable_to_meltdown():
         logger.warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e)
         raise Exception("Command output for failed meltdown check: '%s'" % output)
 
+    logger.debug("Writing results from meltdown checker to cache file, %s" % cache_file)
+    write_to_json(cache_file, CVEs)
     return CVEs[0]["VULNERABLE"]
 
 

src/yunohost/utils/packages.py

@@ -19,6 +19,7 @@
 
 """
 import re
+import os
 import logging
 from collections import OrderedDict
 
@@ -470,3 +471,13 @@ def ynh_packages_version(*args, **kwargs):
         'yunohost', 'yunohost-admin', 'moulinette', 'ssowat',
         with_repo=True
     )
+
+
+def dpkg_is_broken():
+    # If dpkg is broken, /var/lib/dpkg/updates
+    # will contains files like 0001, 0002, ...
+    # ref: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174
+    if not os.path.isdir("/var/lib/dpkg/updates/"):
+        return False
+    return any(re.match("^[0-9]+$", f)
+               for f in os.listdir("/var/lib/dpkg/updates/"))