Merge branch 'stretch-unstable' into helpers_min_version

data/helpers.d/backend

@@ -164,10 +164,10 @@ ynh_remove_systemd_config () {
 
 	local finalsystemdconf="/etc/systemd/system/$service.service"
 	if [ -e "$finalsystemdconf" ]; then
-		sudo systemctl stop $service
+		ynh_systemd_action --service_name=$service --action=stop
-		sudo systemctl disable $service
+		systemctl disable $service
 		ynh_secure_remove --file="$finalsystemdconf"
-		sudo systemctl daemon-reload
+		systemctl daemon-reload
 	fi
 }
 
@@ -234,7 +234,7 @@ ynh_add_nginx_config () {
 
 	ynh_store_file_checksum --file="$finalnginxconf"
 
-	sudo systemctl reload nginx
+	ynh_systemd_action --service_name=nginx --action=reload
 }
 
 # Remove the dedicated nginx config
@@ -244,7 +244,7 @@ ynh_add_nginx_config () {
 # Requires YunoHost version 2.7.2 or higher.
 ynh_remove_nginx_config () {
 	ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf"
-	sudo systemctl reload nginx
+	ynh_systemd_action --service_name=nginx --action=reload
 }
 
 # Create a dedicated php-fpm config
@@ -281,7 +281,7 @@ ynh_add_fpm_config () {
 		sudo chown root: "$finalphpini"
 		ynh_store_file_checksum "$finalphpini"
 	fi
-	sudo systemctl reload $fpm_service
+	ynh_systemd_action --service_name=$fpm_service --action=reload
 }
 
 # Remove the dedicated php-fpm config
@@ -299,7 +299,7 @@ ynh_remove_fpm_config () {
 	fi
 	ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf"
 	ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1
-	sudo systemctl reload $fpm_service
+	ynh_systemd_action --service_name=$fpm_service --action=reload
 }
 
 # Create a dedicated fail2ban config (jail and filter conf files)
@@ -366,6 +366,7 @@ ynh_remove_fpm_config () {
 # Requires YunoHost version 3.?.? or higher.
 ynh_add_fail2ban_config () {
   # Declare an array to define the options of this helper.
+  local legacy_args=lrmptv
   declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=)
   local logpath
   local failregex

data/helpers.d/package

@@ -27,7 +27,7 @@ ynh_wait_dpkg_free() {
             while read dpkg_file <&9
             do
                 # Check if the name of this file contains only numbers.
-                if echo "$dpkg_file" | grep -Pq "^[[:digit:]]*$"
+                if echo "$dpkg_file" | grep -Pq "^[[:digit:]]+$"
                 then
                     # If so, that a remaining of dpkg.
                     ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem."

data/helpers.d/print

@@ -100,6 +100,7 @@ ynh_print_err () {
 # usage: ynh_exec_err command to execute
 # usage: ynh_exec_err "command to execute | following command"
 # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe.
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # | arg: command - command to execute
 #
@@ -113,6 +114,7 @@ ynh_exec_err () {
 # usage: ynh_exec_warn command to execute
 # usage: ynh_exec_warn "command to execute | following command"
 # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe.
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # | arg: command - command to execute
 #
@@ -126,6 +128,7 @@ ynh_exec_warn () {
 # usage: ynh_exec_warn_less command to execute
 # usage: ynh_exec_warn_less "command to execute | following command"
 # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe.
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # | arg: command - command to execute
 #
@@ -139,6 +142,7 @@ ynh_exec_warn_less () {
 # usage: ynh_exec_quiet command to execute
 # usage: ynh_exec_quiet "command to execute | following command"
 # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe.
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # | arg: command - command to execute
 #
@@ -152,6 +156,7 @@ ynh_exec_quiet () {
 # usage: ynh_exec_fully_quiet command to execute
 # usage: ynh_exec_fully_quiet "command to execute | following command"
 # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe.
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # | arg: command - command to execute
 #

data/helpers.d/psql

@@ -1,23 +1,276 @@
+#!/bin/bash
+
+PSQL_ROOT_PWD_FILE=/etc/yunohost/psql
+
+# Open a connection as a user
+#
+# example: ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;"
+# example: ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql
+#
+# usage: ynh_psql_connect_as --user=user --password=password [--database=database]
+# | arg: -u, --user - the user name to connect as
+# | arg: -p, --password - the user password
+# | arg: -d, --database - the database to connect to
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_connect_as() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=upd
+	declare -Ar args_array=([u]=user= [p]=password= [d]=database=)
+	local user
+	local password
+	local database
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	database="${database:-}"
+
+	sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database"
+}
+
+# Execute a command as root user
+#
+# usage: ynh_psql_execute_as_root --sql=sql [--database=database]
+# | arg: -s, --sql - the SQL command to execute
+# | arg: -d, --database - the database to connect to
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_execute_as_root() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=sd
+	declare -Ar args_array=([s]=sql= [d]=database=)
+	local sql
+	local database
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	database="${database:-}"
+
+	ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \
+		--database="$database" <<<"$sql"
+}
+
+# Execute a command from a file as root user
+#
+# usage: ynh_psql_execute_file_as_root --file=file [--database=database]
+# | arg: -f, --file - the file containing SQL commands
+# | arg: -d, --database - the database to connect to
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_execute_file_as_root() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=fd
+	declare -Ar args_array=([f]=file= [d]=database=)
+	local file
+	local database
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	database="${database:-}"
+
+	ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \
+		--database="$database" <"$file"
+}
+
+# Create a database and grant optionnaly privilegies to a user
+#
+# [internal]
+#
+# usage: ynh_psql_create_db db [user]
+# | arg: db - the database name to create
+# | arg: user - the user to grant privilegies
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_create_db() {
+	local db=$1
+	local user=${2:-}
+
+	local sql="CREATE DATABASE ${db};"
+
+	# grant all privilegies to user
+	if [ -n "$user" ]; then
+		sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;"
+	fi
+
+	ynh_psql_execute_as_root --sql="$sql"
+}
+
+# Drop a database
+#
+# [internal]
+#
+# If you intend to drop the database *and* the associated user,
+# consider using ynh_psql_remove_db instead.
+#
+# usage: ynh_psql_drop_db db
+# | arg: db - the database name to drop
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_drop_db() {
+	local db=$1
+	sudo --login --user=postgres dropdb $db
+}
+
+# Dump a database
+#
+# example: ynh_psql_dump_db 'roundcube' > ./dump.sql
+#
+# usage: ynh_psql_dump_db --database=database
+# | arg: -d, --database - the database name to dump
+# | ret: the psqldump output
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_dump_db() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=d
+	declare -Ar args_array=([d]=database=)
+	local database
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	sudo --login --user=postgres pg_dump "$database"
+}
+
+# Create a user
+#
+# [internal]
+#
+# usage: ynh_psql_create_user user pwd
+# | arg: user - the user name to create
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_create_user() {
+	local user=$1
+	local pwd=$2
+	ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'"
+}
+
+# Check if a psql user exists
+#
+# usage: ynh_psql_user_exists --user=user
+# | arg: -u, --user - the user for which to check existence
+ynh_psql_user_exists() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=u
+	declare -Ar args_array=([u]=user=)
+	local user
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user" ; then
+		return 1
+	else
+		return 0
+	fi
+}
+
+# Check if a psql database exists
+#
+# usage: ynh_psql_database_exists --database=database
+# | arg: -d, --database - the database for which to check existence
+ynh_psql_database_exists() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=d
+	declare -Ar args_array=([d]=database=)
+	local database
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$user"; then
+		return 1
+	else
+		return 0
+	fi
+}
+
+# Drop a user
+#
+# [internal]
+#
+# usage: ynh_psql_drop_user user
+# | arg: user - the user name to drop
+#
+# Requires YunoHost version 3.?.? or higher.
+ynh_psql_drop_user() {
+	ynh_psql_execute_as_root --sql="DROP USER ${1};"
+}
+
+# Create a database, an user and its password. Then store the password in the app's config
+#
+# After executing this helper, the password of the created database will be available in $db_pwd
+# It will also be stored as "psqlpwd" into the app settings.
+#
+# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
+# | arg: -u, --db_user - Owner of the database
+# | arg: -n, --db_name - Name of the database
+# | arg: -p, --db_pwd - Password of the database. If not given, a password will be generated
+ynh_psql_setup_db() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=unp
+	declare -Ar args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
+	local db_user
+	local db_name
+	db_pwd=""
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	local new_db_pwd=$(ynh_string_random) # Generate a random password
+	# If $db_pwd is not given, use new_db_pwd instead for db_pwd
+	db_pwd="${db_pwd:-$new_db_pwd}"
+
+	if ! ynh_psql_user_exists --user=$db_user; then
+		ynh_psql_create_user "$db_user" "$db_pwd"
+	fi
+
+	ynh_psql_create_db "$db_name" "$db_user"                     # Create the database
+	ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config
+}
+
+# Remove a database if it exists, and the associated user
+#
+# usage: ynh_psql_remove_db --db_user=user --db_name=name
+# | arg: -u, --db_user - Owner of the database
+# | arg: -n, --db_name - Name of the database
+ynh_psql_remove_db() {
+	# Declare an array to define the options of this helper.
+	local legacy_args=un
+	declare -Ar args_array=([u]=db_user= [n]=db_name=)
+	local db_user
+	local db_name
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE)
+	if ynh_psql_database_exists --database=$db_name; then # Check if the database exists
+		echo "Removing database $db_name" >&2
+		ynh_psql_drop_db $db_name # Remove the database
+	else
+		echo "Database $db_name not found" >&2
+	fi
+
+	# Remove psql user if it exists
+	if ynh_psql_user_exists --user=$db_user; then
+		echo "Removing user $db_user" >&2
+		ynh_psql_drop_user $db_user
+	else
+		echo "User $db_user not found" >&2
+	fi
+}
+
 # Create a master password and set up global settings
 # Please always call this script in install and restore scripts
 #
 # usage: ynh_psql_test_if_first_run
-#
-# Requires YunoHost version 3.?.? or higher.
 ynh_psql_test_if_first_run() {
-	if [ -f /etc/yunohost/psql ];
+	if [ -f "$PSQL_ROOT_PWD_FILE" ]; then
-	then
 		echo "PostgreSQL is already installed, no need to create master password"
 	else
 		local pgsql="$(ynh_string_random)"
 		echo "$pgsql" >/etc/yunohost/psql
 
-		if [ -e /etc/postgresql/9.4/ ]
+		if [ -e /etc/postgresql/9.4/ ]; then
-		then
 			local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf
-		elif [ -e /etc/postgresql/9.6/ ]
+			local logfile=/var/log/postgresql/postgresql-9.4-main.log
-		then
+		elif [ -e /etc/postgresql/9.6/ ]; then
 			local pg_hba=/etc/postgresql/9.6/main/pg_hba.conf
+			local logfile=/var/log/postgresql/postgresql-9.6-main.log
 		else
 			ynh_die "postgresql shoud be 9.4 or 9.6"
 		fi
@@ -29,140 +282,12 @@ ynh_psql_test_if_first_run() {
 		# https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF
 		# Note: we can't use peer since YunoHost create users with nologin
 		#  See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user
-		sed -i '/local\s*all\s*all\s*peer/i \
+		ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3password" --target_file="$pg_hba"
-			local all all password' "$pg_hba"
+
+		# Advertise service in admin panel
+		yunohost service add postgresql --log "$logfile"
+
 		systemctl enable postgresql
 		systemctl reload postgresql
 	fi
 }
-
-# Open a connection as a user
-#
-# example: ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;"
-# example: ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql
-#
-# usage: ynh_psql_connect_as user pwd [db]
-# | arg: user - the user name to connect as
-# | arg: pwd - the user password
-# | arg: db - the database to connect to
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_connect_as() {
-	local user="$1"
-	local pwd="$2"
-	local db="$3"
-	sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$pwd" psql "$db"
-}
-
-# # Execute a command as root user
-#
-# usage: ynh_psql_execute_as_root sql [db]
-# | arg: sql - the SQL command to execute
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_execute_as_root () {
-	local sql="$1"
-	sudo --login --user=postgres psql <<< "$sql"
-}
-
-# Execute a command from a file as root user
-#
-# usage: ynh_psql_execute_file_as_root file [db]
-# | arg: file - the file containing SQL commands
-# | arg: db - the database to connect to
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_execute_file_as_root() {
-	local file="$1"
-	local db="$2"
-	sudo --login --user=postgres psql "$db" < "$file"
-}
-
-# Create a database, an user and its password. Then store the password in the app's config
-#
-# After executing this helper, the password of the created database will be available in $db_pwd
-# It will also be stored as "psqlpwd" into the app settings.
-#
-# usage: ynh_psql_setup_db user name [pwd]
-# | arg: user - Owner of the database
-# | arg: name - Name of the database
-# | arg: pwd - Password of the database. If not given, a password will be generated
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_setup_db () {
-	local db_user="$1"
-	local db_name="$2"
-	local new_db_pwd=$(ynh_string_random)	# Generate a random password
-	# If $3 is not given, use new_db_pwd instead for db_pwd.
-	local db_pwd="${3:-$new_db_pwd}"
-	ynh_psql_create_db "$db_name" "$db_user" "$db_pwd"	# Create the database
-	ynh_app_setting_set "$app" psqlpwd "$db_pwd"	# Store the password in the app's config
-}
-
-# Create a database and grant privilegies to a user
-#
-# usage: ynh_psql_create_db db [user [pwd]]
-# | arg: db - the database name to create
-# | arg: user - the user to grant privilegies
-# | arg: pwd  - the user password
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_create_db() {
-	local db="$1"
-	local user="$2"
-	local pwd="$3"
-	ynh_psql_create_user "$user" "$pwd"
-	sudo --login --user=postgres createdb --owner="$user" "$db"
-}
-
-# Drop a database
-#
-# usage: ynh_psql_drop_db db
-# | arg: db - the database name to drop
-# | arg: user - the user to drop
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_remove_db() {
-	local db="$1"
-	local user="$2"
-	sudo --login --user=postgres dropdb "$db"
-	ynh_psql_drop_user "$user"
-}
-
-# Dump a database
-#
-# example: ynh_psql_dump_db 'roundcube' > ./dump.sql
-#
-# usage: ynh_psql_dump_db db
-# | arg: db - the database name to dump
-# | ret: the psqldump output
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_dump_db() {
-	local db="$1"
-	sudo --login --user=postgres pg_dump "$db"
-}
-
-
-# Create a user
-#
-# usage: ynh_psql_create_user user pwd [host]
-# | arg: user - the user name to create
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_create_user() {
-	local user="$1"
-	local pwd="$2"
-	sudo --login --user=postgres psql -c"CREATE USER $user WITH PASSWORD '$pwd'" postgres
-}
-
-# Drop a user
-#
-# usage: ynh_psql_drop_user user
-# | arg: user - the user name to drop
-#
-# Requires YunoHost version 3.?.? or higher.
-ynh_psql_drop_user() {
-	local user="$1"
-	sudo --login --user=postgres dropuser "$user"
-}

data/helpers.d/setting

@@ -16,7 +16,7 @@ ynh_app_setting_get() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    sudo yunohost app setting "$app" "$key" --output-as plain --quiet
+    ynh_app_setting "get" "$app" "$key"
 }
 
 # Set an application setting
@@ -37,7 +37,7 @@ ynh_app_setting_set() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    sudo yunohost app setting "$app" "$key" --value="$value" --quiet
+    ynh_app_setting "set" "$app" "$key" "$value"
 }
 
 # Delete an application setting
@@ -56,5 +56,38 @@ ynh_app_setting_delete() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    sudo yunohost app setting -d "$app" "$key" --quiet
+    ynh_app_setting "delete" "$app" "$key"
+}
+
+# Small "hard-coded" interface to avoid calling "yunohost app" directly each
+# time dealing with a setting is needed (which may be so slow on ARM boards)
+#
+# [internal]
+#
+ynh_app_setting()
+{
+    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python - <<EOF
+import os, yaml
+app, action = os.environ['APP'], os.environ['ACTION'].lower()
+key, value = os.environ['KEY'], os.environ.get('VALUE', None)
+setting_file = "/etc/yunohost/apps/%s/settings.yml" % app
+assert os.path.exists(setting_file), "Setting file %s does not exists ?" % setting_file
+with open(setting_file) as f:
+    settings = yaml.load(f)
+if action == "get":
+    if key in settings:
+        print(settings[key])
+else:
+    if action == "delete":
+        if key in settings:
+            del settings[key]
+    elif action == "set":
+        if key in ['redirected_urls', 'redirected_regex']:
+            value = yaml.load(value)
+        settings[key] = value
+    else:
+        raise ValueError("action should either be get, set or delete")
+    with open(setting_file, "w") as f:
+        yaml.safe_dump(settings, f, default_flow_style=False)
+EOF
 }

data/helpers.d/system

@@ -60,6 +60,112 @@ ynh_get_debian_release () {
 	echo $(lsb_release --codename --short)
 }
 
+# Start (or other actions) a service,  print a log in case of failure and optionnaly wait until the service is completely started
+#
+# usage: ynh_systemd_action [-n service_name] [-a action] [ [-l "line to match"] [-p log_path] [-t timeout] [-e length] ]
+# | arg: -n, --service_name= - Name of the service to start. Default : $app
+# | arg: -a, --action=       - Action to perform with systemctl. Default: start
+# | arg: -l, --line_match=   - Line to match - The line to find in the log to attest the service have finished to boot.
+#                              If not defined it don't wait until the service is completely started.
+#                              WARNING: When using --line_match, you should always add `ynh_clean_check_starting` into your
+#                                `ynh_clean_setup` at the beginning of the script. Otherwise, tail will not stop in case of failure
+#                                of the script. The script will then hang forever.
+# | arg: -p, --log_path=     - Log file - Path to the log file. Default : /var/log/$app/$app.log
+# | arg: -t, --timeout=      - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds.
+# | arg: -e, --length=       - Length of the error log : Default : 20
+ynh_systemd_action() {
+    # Declare an array to define the options of this helper.
+    declare -Ar args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= )
+    local service_name
+    local action
+    local line_match
+    local length
+    local log_path
+    local timeout
+
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local service_name="${service_name:-$app}"
+    local action=${action:-start}
+    local log_path="${log_path:-/var/log/$service_name/$service_name.log}"
+    local length=${length:-20}
+    local timeout=${timeout:-300}
+
+    # Start to read the log
+    if [[ -n "${line_match:-}" ]]
+    then
+        local templog="$(mktemp)"
+        # Following the starting of the app in its log
+        if [ "$log_path" == "systemd" ] ; then
+            # Read the systemd journal
+            journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" &
+            # Get the PID of the journalctl command
+            local pid_tail=$!
+        else
+            # Read the specified log file
+            tail -F -n0 "$log_path" > "$templog" 2>&1 &
+            # Get the PID of the tail command
+            local pid_tail=$!
+        fi
+    fi
+
+    ynh_print_info --message="${action^} the service $service_name"
+
+    # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running.
+    if [ "$action" == "reload" ]; then
+        action="reload-or-restart"
+    fi
+
+    systemctl $action $service_name \
+        || ( journalctl --no-pager --lines=$length -u $service_name >&2 \
+        ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \
+        ; false )
+
+    # Start the timeout and try to find line_match
+    if [[ -n "${line_match:-}" ]]
+    then
+        local i=0
+        for i in $(seq 1 $timeout)
+        do
+            # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout
+            if grep --quiet "$line_match" "$templog"
+            then
+                ynh_print_info --message="The service $service_name has correctly started."
+                break
+            fi
+            if [ $i -eq 3 ]; then
+                echo -n "Please wait, the service $service_name is ${action}ing" >&2
+            fi
+            if [ $i -ge 3 ]; then
+                echo -n "." >&2
+            fi
+            sleep 1
+        done
+        if [ $i -ge 3 ]; then
+            echo "" >&2
+        fi
+        if [ $i -eq $timeout ]
+        then
+            ynh_print_warn --message="The service $service_name didn't fully started before the timeout."
+            ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:"
+            journalctl --no-pager --lines=$length -u $service_name >&2
+            test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2
+        fi
+        ynh_clean_check_starting
+    fi
+}
+
+# Clean temporary process and file used by ynh_check_starting
+# (usually used in ynh_clean_setup scripts)
+#
+# usage: ynh_clean_check_starting
+ynh_clean_check_starting () {
+	# Stop the execution of tail.
+	kill -s 15 $pid_tail 2>&1
+	ynh_secure_remove "$templog" 2>&1
+}
+
 # Read the value of a key in a ynh manifest file
 #
 # usage: ynh_read_manifest manifest key

data/helpers.d/user

@@ -71,6 +71,21 @@ ynh_system_user_exists() {
     getent passwd "$username" &>/dev/null
 }
 
+# Check if a group exists on the system
+#
+# usage: ynh_system_group_exists --group=group
+# | arg: -g, --group - the group to check
+ynh_system_group_exists() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=g
+    declare -Ar args_array=( [g]=group= )
+    local group
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    getent group "$group" &>/dev/null
+}
+
 # Create a system user
 #
 # examples:
@@ -128,11 +143,19 @@ ynh_system_user_delete () {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ynh_system_user_exists "$username"	# Check if the user exists on the system
+    # Check if the user exists on the system
+    if ynh_system_user_exists "$username"
     then
 		echo "Remove the user $username" >&2
-		sudo userdel $username
+		deluser $username
 	else
 		echo "The user $username was not found" >&2
     fi
+
+    # Check if the group exists on the system
+    if ynh_system_group_exists "$username"
+    then
+		echo "Remove the group $username" >&2
+		delgroup $username
+    fi
 }

data/templates/dnsmasq/plain/resolv.dnsmasq.conf

@@ -9,25 +9,37 @@
 
 # (FR) FDN
 nameserver 80.67.169.12
+nameserver 2001:910:800::12
 nameserver 80.67.169.40
+nameserver 2001:910:800::40
 # (FR) LDN
 nameserver 80.67.188.188
+nameserver 2001:913::8
 # (FR) ARN
 nameserver 89.234.141.66
+nameserver 2a00:5881:8100:1000::3
 # (FR) Aquilenet
 nameserver 185.233.100.100
+nameserver 2a0c:e300::100
 nameserver 185.233.100.101
+nameserver 2a0c:e300::101
 # (FR) gozmail / grifon
 nameserver 80.67.190.200
+nameserver 2a00:5884:8218::1
 # (DE) FoeBud / Digital Courage
 nameserver 85.214.20.141
 # (DE) CCC Berlin
 nameserver 195.160.173.53
 # (DE) AS250
 nameserver 194.150.168.168
+nameserver 2001:4ce8::53
 # (DE) Ideal-Hosting
 nameserver 84.200.69.80
+nameserver 2001:1608:10:25::1c04:b12f
 nameserver 84.200.70.40
+nameserver 2001:1608:10:25::9249:d69b
 # (DK) censurfridns
 nameserver 91.239.100.100
+nameserver 2001:67c:28a4::
 nameserver 89.233.43.71
+nameserver 2002:d596:2a92:1:71:53::

data/templates/nginx/plain/global.conf

@@ -1,2 +1 @@
 server_tokens off;
-gzip_types text/css text/javascript application/javascript;

data/templates/nginx/plain/yunohost_admin.conf

@@ -51,6 +51,10 @@ server {
     more_set_headers "X-Permitted-Cross-Domain-Policies : none";
     more_set_headers "X-Frame-Options : SAMEORIGIN";
 
+    # Disable gzip to protect against BREACH
+    # Read https://trac.nginx.org/nginx/ticket/1720 (text/html cannot be disabled!)
+    gzip off;
+
     location / {
         return 302 https://$http_host/yunohost/admin;
     }

data/templates/nginx/server.tpl.conf

@@ -71,6 +71,10 @@ server {
     resolver_timeout 5s;
     {% endif %}
 
+    # Disable gzip to protect against BREACH
+    # Read https://trac.nginx.org/nginx/ticket/1720 (text/html cannot be disabled!)
+    gzip off;
+
     access_by_lua_file /usr/share/ssowat/access.lua;
 
     include /etc/nginx/conf.d/{{ domain }}.d/*.conf;

debian/changelog

@@ -1,3 +1,69 @@
+yunohost (3.5.0.2) testing; urgency=low
+
+  - [fix] Make sure that `ynh_system_user_delete` also deletes the group (#680)
+  - [enh] `ynh_systemd_action` : reload-or-restart instead of just reload (#681)
+
+  Last minute fixes by Maniack ;)
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Thu, 14 Mar 2019 03:45:00  +0000
+
+yunohost (3.5.0.1) testing; urgency=low
+
+  - [fix] #675 introduced a bug in nginx conf ...
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Wed, 13 Mar 2019 19:23:00  +0000
+
+yunohost (3.5.0) testing; urgency=low
+
+  Core
+  ----
+
+  - [fix] Disable gzip entirely to avoid BREACH attacks (#675)
+  - [fix] Backup tests were broken (#673)
+  - [fix] Backup fails because output directory not empty (#672)
+  - [fix] Reject app password if they contains { or } (#671)
+  - [enh] Allow `display_text` 'fake' argument in manifest.json (#669)
+  - [fix] Optimize dyndns requests (#662)
+  - [enh] Don't add Strict-Transport-Security header in nginx conf if using a selfsigned cert (#661)
+  - [enh] Add apt-transport-https to dependencies (#658)
+  - [enh] Cache results from meltdown vulnerability checker (#656)
+  - [enh] Ensure the tar file is closed during the backup (#655)
+  - [enh] Be able to define hook to trigger when changing a setting (#654)
+  - [enh] Assert dpkg is not broken before app install (#652)
+  - [fix] Loading only one helper file leads to errors because missing getopts (#651)
+  - [enh] Improve / add some messages to improve UX (#650)
+  - [enh] Reload fail2ban instead of restart (#649)
+  - [enh] Add IPv6 resolvers from diyisp.org to resolv.dnsmasq.conf (#639)
+  - [fix] Remove old SMTP port (465) from fail2ban jail.conf (#637)
+  - [enh] Improve protection against indexation from the robots. (#622)
+  - [enh] Allow hooks to return data (#526)
+  - [fix] Do not make version number available from web API to unauthenticated users (#291)
+  - [i18n] Improve Russian and Chinese (Mandarin) translations
+
+  App helpers
+  -----------
+
+  - [enh] Optimize app setting helpers (#663, #676)
+  - [enh] Handle `ynh_install_nodejs` for arm64 / aarch64 (#660)
+  - [enh] Update postgresql helpers (#657)
+  - [enh] Print diff of files when backup by `ynh_backup_if_checksum_is_different` (#648)
+  - [enh] Add app debugger helper (#647)
+  - [fix] Escape double quote before eval in getopts (#646)
+  - [fix] `ynh_local_curl` not using the right url in some cases (#644)
+  - [fix] Get rid of annoying 'unable to initialize frontend' messages (#643)
+  - [enh] Check if dpkg is not broken when calling `ynh_wait_dpkg_free` (#638)
+  - [enh] Warn the packager that `ynh_secure_remove` should be used with only one arg… (#635, #642)
+  - [enh] Add `ynh_script_progression` helper (#634)
+  - [enh] Add `ynh_systemd_action` helper (#633)
+  - [enh] Allow to dig deeper into an archive with `ynh_setup_source` (#630)
+  - [enh] Use getops (#561)
+  - [enh] Add `ynh_check_app_version_changed` helper (#521)
+  - [enh] Add fail2ban helpers (#364)
+
+  Contributors: Alexandre Aubin, Jimmy Monin, Josué Tille, Kayou, Laurent Peuch, Lukas Fülling, Maniack Crudelis, Taekiro, frju365, ljf, opi, yalh76, Алексей
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Wed, 13 Mar 2019 16:10:00  +0000
+
 yunohost (3.4.2.4) stable; urgency=low
 
   - [fix] Meltdown vulnerability checker something outputing trash instead of pure json

locales/ca.json

@@ -21,5 +21,64 @@
     "app_location_already_used": "L'aplicació '{app}' ja està instal·lada en aquest camí ({path})",
     "app_make_default_location_already_used": "No es pot fer l'aplicació '{app}' per defecte en el domini {domain} ja que ja és utilitzat per una altra aplicació '{other_app}'",
     "app_location_install_failed": "No s'ha pogut instal·lar l'aplicació en aquest camí ja que entra en conflicte amb l'aplicació '{other_app}' ja instal·lada a '{other_path}'",
-    "app_location_unavailable": "Aquesta url no està disponible o entra en conflicte amb aplicacions ja instal·lades:\n{apps:s}"
+    "app_location_unavailable": "Aquesta url no està disponible o entra en conflicte amb aplicacions ja instal·lades:\n{apps:s}",
+    "app_manifest_invalid": "Manifest d'aplicació incorrecte: {error}",
+    "app_no_upgrade": "No hi ha cap aplicació per actualitzar",
+    "app_not_correctly_installed": "{app:s} sembla estar mal instal·lada",
+    "app_not_installed": "{app:s} no està instal·lada",
+    "app_not_properly_removed": "{app:s} no s'ha pogut suprimir correctament",
+    "app_package_need_update": "El paquet de l'aplicació {app} ha de ser actualitzat per poder seguir els canvis de YunoHost",
+    "app_removed": "{app:s} ha estat suprimida",
+    "app_requirements_checking": "Verificació dels paquets requerits per {app}",
+    "app_requirements_failed": "No es poden satisfer els requeriments per {app}: {error}",
+    "app_requirements_unmeet": "No es compleixen els requeriments per {app}, el paquet {pkgname} ({version}) ha de ser {spec}",
+    "app_sources_fetch_failed": "No s'han pogut carregar els fitxers font",
+    "app_unknown": "Aplicació desconeguda",
+    "app_unsupported_remote_type": "El tipus remot utilitzat per l'aplicació no està suportat",
+    "app_upgrade_app_name": "Actualitzant l'aplicació {app}...",
+    "app_upgrade_failed": "No s'ha pogut actualitzar {app:s}",
+    "app_upgrade_some_app_failed": "No s'han pogut actualitzar algunes aplicacions",
+    "app_upgraded": "{app:s} ha estat actualitzada",
+    "appslist_corrupted_json": "No s'han pogut carregar les llistes d'aplicacions. Sembla que {filename:s} està danyat.",
+    "appslist_could_not_migrate": "No s'ha pogut migrar la llista d'aplicacions {appslist:s}! No s'ha pogut analitzar la URL... L'antic cronjob s'ha guardat a {bkp_file:s}.",
+    "appslist_fetched": "S'ha descarregat la llista d'aplicacions {appslist:s} correctament",
+    "appslist_migrating": "Migrant la llista d'aplicacions {appslist:s} ...",
+    "appslist_name_already_tracked": "Ja hi ha una llista d'aplicacions registrada amb el nom {name:s}.",
+    "appslist_removed": "S'ha eliminat la llista d'aplicacions {appslist:s}",
+    "appslist_retrieve_bad_format": "L'arxiu obtingut per la llista d'aplicacions {appslist:s} no és vàlid",
+    "appslist_retrieve_error": "No s'ha pogut obtenir la llista d'aplicacions remota {appslist:s}: {error:s}",
+    "appslist_unknown": "La llista d'aplicacions {appslist:s} es desconeguda.",
+    "appslist_url_already_tracked": "Ja hi ha una llista d'aplicacions registrada amb al URL {url:s}.",
+    "ask_current_admin_password": "Contrasenya d'administrador actual",
+    "ask_email": "Correu electrònic",
+    "ask_firstname": "Nom",
+    "ask_lastname": "Cognom",
+    "ask_list_to_remove": "Llista per a suprimir",
+    "ask_main_domain": "Domini principal",
+    "ask_new_admin_password": "Nova contrasenya d'administrador",
+    "ask_password": "Contrasenya",
+    "ask_path": "Camí",
+    "backup_abstract_method": "Encara no s'ha implementat aquest mètode de copia de seguretat",
+    "backup_action_required": "S'ha d'especificar què s'ha de guardar",
+    "backup_app_failed": "No s'ha pogut fer la còpia de seguretat de l'aplicació \"{app:s}\"",
+    "backup_applying_method_borg": "Enviant tots els fitxers de la còpia de seguretat al repositori borg-backup...",
+    "backup_applying_method_copy": "Còpia de tots els fitxers a la còpia de seguretat...",
+    "backup_applying_method_custom": "Crida del mètode de còpia de seguretat personalitzat \"{method:s}\"...",
+    "backup_applying_method_tar": "Creació de l'arxiu tar de la còpia de seguretat...",
+    "backup_archive_app_not_found": "L'aplicació \"{app:s}\" no es troba dins l'arxiu de la còpia de seguretat",
+    "backup_archive_broken_link": "No s'ha pogut accedir a l'arxiu de la còpia de seguretat (enllaç invàlid cap a {path:s})",
+    "backup_archive_mount_failed": "No s'ha pogut carregar l'arxiu de la còpia de seguretat",
+    "backup_archive_name_exists": "Ja hi ha una còpia de seguretat amb aquest nom",
+    "backup_archive_name_unknown": "Còpia de seguretat local \"{name:s}\" desconeguda",
+    "backup_archive_open_failed": "No s'ha pogut obrir l'arxiu de la còpia de seguretat",
+    "backup_archive_system_part_not_available": "La part \"{part:s}\" del sistema no està disponible en aquesta copia de seguretat",
+    "backup_archive_writing_error": "No es poden afegir arxius a l'arxiu comprimit de la còpia de seguretat",
+    "backup_ask_for_copying_if_needed": "Alguns fitxers no s'han pogut preparar per la còpia de seguretat utilitzant el mètode que evita malgastar espai del sistema temporalment. Per fer la còpia de seguretat, s'han d'utilitzar {size:s}MB temporalment. Hi esteu d'acord?",
+    "backup_borg_not_implemented": "El mètode de còpia de seguretat Borg encara no està implementat",
+    "backup_cant_mount_uncompress_archive": "No es pot carregar en mode de lectura només el directori de l'arxiu descomprimit",
+    "backup_cleaning_failed": "No s'ha pogut netejar el directori temporal de la còpia de seguretat",
+    "backup_copying_to_organize_the_archive": "Copiant {size:s}MB per organitzar l'arxiu",
+    "backup_couldnt_bind": "No es pot lligar {src:s} amb {dest:s}.",
+    "backup_created": "S'ha creat la còpia de seguretat",
+    "backup_creating_archive": "Creant l'arxiu de la còpia de seguretat"
 }

locales/el.json

@@ -0,0 +1 @@
+{}

locales/en.json

@@ -218,7 +218,8 @@
     "good_practices_about_admin_password": "You are now about to define a new administration password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).",
     "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).",
     "hook_exec_failed": "Script execution failed: {path:s}",
-    "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}",
+    "hook_exec_not_terminated": "Script execution did not finish properly: {path:s}",
+    "hook_json_return_error": "Failed to read return from hook {path:s}. Error: {msg:s}. Raw content: {raw_content}",
     "hook_list_by_invalid": "Invalid property to list hook by",
     "hook_name_unknown": "Unknown hook name '{name:s}'",
     "installation_complete": "Installation complete",
@@ -380,6 +381,7 @@
     "pattern_port_or_range": "Must be a valid port number (i.e. 0-65535) or range of ports (e.g. 100:200)",
     "pattern_positive_number": "Must be a positive number",
     "pattern_username": "Must be lower-case alphanumeric and underscore characters only",
+    "pattern_password_app": "Sorry, passwords should not contain the following characters: {forbidden_chars}",
     "port_already_closed": "Port {port:d} is already closed for {ip_version:s} connections",
     "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections",
     "port_available": "Port {port:d} is available",

locales/pl.json

@@ -0,0 +1 @@
+{}

locales/ru.json

@@ -6,5 +6,41 @@
     "app_already_installed": "{app:s} уже установлено",
     "app_already_installed_cant_change_url": "Это приложение уже установлено. URL не может быть изменен только с помощью этой функции. Изучите `app changeurl`, если это доступно.",
     "app_argument_choice_invalid": "Неверный выбор для аргумента '{name:s}', Это должно быть  '{choices:s}'",
-    "app_argument_invalid": "Недопустимое значение аргумента '{name:s}': {error:s}'"
+    "app_argument_invalid": "Недопустимое значение аргумента '{name:s}': {error:s}'",
+    "app_already_up_to_date": "{app:s} уже обновлено",
+    "app_argument_required": "Аргумент '{name:s}' необходим",
+    "app_change_no_change_url_script": "Приложение {app_name:s} не поддерживает изменение URL, вы должны обновить его.",
+    "app_change_url_identical_domains": "Старый и новый domain/url_path идентичны ('{domain:s}{path:s}'), ничего делать не надо.",
+    "app_change_url_no_script": "Приложение '{app_name:s}' не поддерживает изменение url. Наверное, вам нужно обновить приложение.",
+    "app_change_url_success": "Успешно изменён {app:s} url на {domain:s}{path:s}",
+    "app_extraction_failed": "Невозможно извлечь файлы для инсталляции",
+    "app_id_invalid": "Неправильный id приложения",
+    "app_incompatible": "Приложение {app} несовместимо с вашей версией YonoHost",
+    "app_install_files_invalid": "Неправильные файлы инсталляции",
+    "app_location_already_used": "Приложение '{app}' уже установлено по этому адресу ({path})",
+    "app_location_install_failed": "Невозможно установить приложение в это место, потому что оно конфликтует с приложением, '{other_app}' установленном на '{other_path}'",
+    "app_location_unavailable": "Этот url отсутствует  или конфликтует с уже установленным приложением или приложениями: {apps:s}",
+    "app_manifest_invalid": "Недопустимый манифест приложения: {error}",
+    "app_no_upgrade": "Нет приложений, требующих обновления",
+    "app_not_correctly_installed": "{app:s} , кажется, установлены неправильно",
+    "app_not_installed": "{app:s} не установлены",
+    "app_not_properly_removed": "{app:s} удалены неправильно",
+    "app_package_need_update": "Пакет приложения  {app} должен быть обновлён в соответствии с изменениями YonoHost",
+    "app_removed": "{app:s} удалено",
+    "app_requirements_checking": "Проверяю необходимые пакеты для {app}...",
+    "app_sources_fetch_failed": "Невозможно получить исходные файлы",
+    "app_unknown": "Неизвестное приложение",
+    "app_upgrade_app_name": "Обновление приложения {app}...",
+    "app_upgrade_failed": "Невозможно обновить {app:s}",
+    "app_upgrade_some_app_failed": "Невозможно обновить некоторые приложения",
+    "app_upgraded": "{app:s} обновлено",
+    "appslist_corrupted_json": "Не могу загрузить список приложений. Кажется,  {filename:s} поврежден.",
+    "appslist_fetched": "Был выбран список приложений {appslist:s}",
+    "appslist_name_already_tracked": "Уже есть зарегистрированный список приложений по имени {name:s}.",
+    "appslist_removed": "Список приложений {appslist:s} удалён",
+    "appslist_retrieve_bad_format": "Неверный файл списка приложений{appslist:s}",
+    "appslist_retrieve_error": "Невозможно получить список удаленных приложений {appslist:s}: {error:s}",
+    "appslist_unknown": "Список приложений {appslist:s} неизвестен.",
+    "appslist_url_already_tracked": "Это уже зарегистрированный список приложений с url{url:s}.",
+    "installation_complete": "Установка завершена"
 }

locales/sv.json

@@ -0,0 +1 @@
+{}

locales/zh_Hans.json

@@ -0,0 +1 @@
+{}

src/yunohost/app.py

@@ -97,7 +97,7 @@ def app_fetchlist(url=None, name=None):
         name -- Name of the list
         url -- URL of remote JSON list
     """
-    if not url.endswith(".json"):
+    if url and not url.endswith(".json"):
         raise YunohostError("This is not a valid application list url. It should end with .json.")
 
     # If needed, create folder where actual appslists are stored
@@ -523,7 +523,7 @@ def app_change_url(operation_logger, auth, app, domain, path):
     os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts", "change_url")))
 
     if hook_exec(os.path.join(APP_TMP_FOLDER, 'scripts/change_url'),
-                 args=args_list, env=env_dict) != 0:
+                 args=args_list, env=env_dict)[0] != 0:
         msg = "Failed to change '%s' url." % app
         logger.error(msg)
         operation_logger.error(msg)
@@ -583,28 +583,28 @@ def app_upgrade(auth, app=[], url=None, file=None):
     not_upgraded_apps = []
 
     apps = app
-    user_specified_list = True
     # If no app is specified, upgrade all apps
     if not apps:
+        # FIXME : not sure what's supposed to happen if there is a url and a file but no apps...
         if not url and not file:
             apps = [app["id"] for app in app_list(installed=True)["apps"]]
-            user_specified_list = False
     elif not isinstance(app, list):
         apps = [app]
 
     # Remove possible duplicates
-    apps = [app for i,app in enumerate(apps) if apps not in L[:i]]
+    apps = [app for i,app in enumerate(apps) if apps not in apps[:i]]
+
+    # Abort if any of those app is in fact not installed..
+    for app in [app for app in apps if not _is_installed(app)]:
+        raise YunohostError('app_not_installed', app=app)
 
     if len(apps) == 0:
         raise YunohostError('app_no_upgrade')
     if len(apps) > 1:
-        logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(app)))
+        logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(apps)))
 
     for app_instance_name in apps:
         logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name))
-        installed = _is_installed(app_instance_name)
-        if not installed:
-            raise YunohostError('app_not_installed', app=app_instance_name)
 
         app_dict = app_info(app_instance_name, raw=True)
 
@@ -618,7 +618,6 @@ def app_upgrade(auth, app=[], url=None, file=None):
         elif app_dict["upgradable"] == "yes":
             manifest, extracted_app_folder = _fetch_app_from_git(app_instance_name)
         else:
-            if user_specified_list:
             logger.success(m18n.n('app_already_up_to_date', app=app_instance_name))
             continue
 
@@ -655,7 +654,7 @@ def app_upgrade(auth, app=[], url=None, file=None):
         # Execute App upgrade script
         os.system('chown -hR admin: %s' % INSTALL_TMP)
         if hook_exec(extracted_app_folder + '/scripts/upgrade',
-                     args=args_list, env=env_dict) != 0:
+                     args=args_list, env=env_dict)[0] != 0:
             msg = m18n.n('app_upgrade_failed', app=app_instance_name)
             not_upgraded_apps.append(app_instance_name)
             logger.error(msg)
@@ -848,7 +847,7 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on
         install_retcode = hook_exec(
             os.path.join(extracted_app_folder, 'scripts/install'),
             args=args_list, env=env_dict
-        )
+        )[0]
     except (KeyboardInterrupt, EOFError):
         install_retcode = -1
     except Exception:
@@ -873,7 +872,7 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on
                 remove_retcode = hook_exec(
                     os.path.join(extracted_app_folder, 'scripts/remove'),
                     args=[app_instance_name], env=env_dict_remove
-                )
+                )[0]
                 if remove_retcode != 0:
                     msg = m18n.n('app_not_properly_removed',
                                  app=app_instance_name)
@@ -964,7 +963,7 @@ def app_remove(operation_logger, auth, app):
     operation_logger.flush()
 
     if hook_exec('/tmp/yunohost_remove/scripts/remove', args=args_list,
-                 env=env_dict) == 0:
+                 env=env_dict)[0] == 0:
         logger.success(m18n.n('app_removed', app=app))
 
         hook_callback('post_app_remove', args=args_list, env=env_dict)
@@ -1563,7 +1562,7 @@ def app_action_run(app, action, args=None):
         env=env_dict,
         chdir=cwd,
         user=action_declaration.get("user", "root"),
-    )
+    )[0]
 
     if retcode not in action_declaration.get("accepted_return_codes", [0]):
         raise YunohostError("Error while executing action '%s' of app '%s': return code %s" % (action, app, retcode), raw_msg=True)
@@ -2203,6 +2202,11 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None):
         if arg_type == 'boolean':
             arg_default = 1 if arg_default else 0
 
+        # do not print for webadmin
+        if arg_type == 'display_text' and msettings.get('interface') != 'api':
+            print(arg["text"])
+            continue
+
         # Attempt to retrieve argument value
         if arg_name in args:
             arg_value = args[arg_name]
@@ -2288,6 +2292,9 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None):
                 else:
                     raise YunohostError('app_argument_choice_invalid', name=arg_name, choices='yes, no, y, n, 1, 0')
         elif arg_type == 'password':
+            forbidden_chars = "{}"
+            if any(char in arg_value for char in forbidden_chars):
+                raise YunohostError('pattern_password_app', forbidden_chars=forbidden_chars)
             from yunohost.utils.password import assert_password_is_strong_enough
             assert_password_is_strong_enough('user', arg_value)
         args_dict[arg_name] = arg_value

src/yunohost/backup.py

@@ -326,10 +326,19 @@ class BackupManager():
         if not os.path.isdir(self.work_dir):
             filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin')
         elif self.is_tmp_work_dir:
-            logger.debug("temporary directory for backup '%s' already exists",
+
+            logger.debug("temporary directory for backup '%s' already exists... attempting to clean it",
                          self.work_dir)
-            # FIXME May be we should clean the workdir here
+
+            # Try to recursively unmount stuff (from a previously failed backup ?)
+            if not _recursive_umount(self.work_dir):
                 raise YunohostError('backup_output_directory_not_empty')
+            else:
+                # If umount succeeded, remove the directory (we checked that
+                # we're in /home/yunohost.backup/tmp so that should be okay...
+                # c.f. method clean() which also does this)
+                filesystem.rm(self.work_dir, recursive=True, force=True)
+                filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin')
 
     #
     # Backup target management                                              #
@@ -593,8 +602,15 @@ class BackupManager():
                             env=env_dict,
                             chdir=self.work_dir)
 
-        if ret["succeed"] != []:
+        ret_succeed = {hook: {path:result["state"] for path, result in infos.items()}
-            self.system_return = ret["succeed"]
+                       for hook, infos in ret.items()
+                       if any(result["state"] == "succeed" for result in infos.values())}
+        ret_failed = {hook: {path:result["state"] for path, result in infos.items.items()}
+                      for hook, infos in ret.items()
+                      if any(result["state"] == "failed" for result in infos.values())}
+
+        if ret_succeed.keys() != []:
+            self.system_return = ret_succeed
 
         # Add files from targets (which they put in the CSV) to the list of
         # files to backup
@@ -610,7 +626,7 @@ class BackupManager():
 
         restore_hooks = hook_list("restore")["hooks"]
 
-        for part in ret['succeed'].keys():
+        for part in ret_succeed.keys():
             if part in restore_hooks:
                 part_restore_hooks = hook_info("restore", part)["hooks"]
                 for hook in part_restore_hooks:
@@ -620,7 +636,7 @@ class BackupManager():
                 logger.warning(m18n.n('restore_hook_unavailable', hook=part))
                 self.targets.set_result("system", part, "Warning")
 
-        for part in ret['failed'].keys():
+        for part in ret_failed.keys():
             logger.error(m18n.n('backup_system_part_failed', part=part))
             self.targets.set_result("system", part, "Error")
 
@@ -682,7 +698,7 @@ class BackupManager():
             subprocess.call(['install', '-Dm555', app_script, tmp_script])
 
             hook_exec(tmp_script, args=[tmp_app_bkp_dir, app],
-                      raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict)
+                      raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict)[0]
 
             self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"])
         except:
@@ -904,7 +920,7 @@ class RestoreManager():
             ret = subprocess.call(["umount", self.work_dir])
             if ret != 0:
                 logger.warning(m18n.n('restore_cleaning_failed'))
-        filesystem.rm(self.work_dir, True, True)
+        filesystem.rm(self.work_dir, recursive=True, force=True)
 
     #
     # Restore target manangement                                            #
@@ -1177,16 +1193,21 @@ class RestoreManager():
                             env=env_dict,
                             chdir=self.work_dir)
 
-        for part in ret['succeed'].keys():
+        ret_succeed = [hook for hook, infos in ret.items()
+                       if any(result["state"] == "succeed" for result in infos.values())]
+        ret_failed = [hook for hook, infos in ret.items()
+                      if any(result["state"] == "failed" for result in infos.values())]
+
+        for part in ret_succeed:
             self.targets.set_result("system", part, "Success")
 
         error_part = []
-        for part in ret['failed'].keys():
+        for part in ret_failed:
             logger.error(m18n.n('restore_system_part_failed', part=part))
             self.targets.set_result("system", part, "Error")
             error_part.append(part)
 
-        if ret['failed']:
+        if ret_failed:
             operation_logger.error(m18n.n('restore_system_part_failed', part=', '.join(error_part)))
         else:
             operation_logger.success()
@@ -1301,7 +1322,7 @@ class RestoreManager():
                       args=[app_backup_in_archive, app_instance_name],
                       chdir=app_backup_in_archive,
                       raise_on_error=True,
-                      env=env_dict)
+                      env=env_dict)[0]
         except:
             msg = m18n.n('restore_app_failed', app=app_instance_name)
             logger.exception(msg)
@@ -1326,7 +1347,7 @@ class RestoreManager():
             # Execute remove script
             # TODO: call app_remove instead
             if hook_exec(remove_script, args=[app_instance_name],
-                         env=env_dict_remove) != 0:
+                         env=env_dict_remove)[0] != 0:
                 msg = m18n.n('app_not_properly_removed', app=app_instance_name)
                 logger.warning(msg)
                 operation_logger.error(msg)
@@ -1514,34 +1535,12 @@ class BackupMethod(object):
                                   directories of the working directories
         """
         if self.need_mount():
-            if self._recursive_umount(self.work_dir) > 0:
+            if not _recursive_umount(self.work_dir):
                 raise YunohostError('backup_cleaning_failed')
 
         if self.manager.is_tmp_work_dir:
             filesystem.rm(self.work_dir, True, True)
 
-    def _recursive_umount(self, directory):
-        """
-        Recursively umount sub directories of a directory
-
-        Args:
-            directory -- a directory path
-        """
-        mount_lines = subprocess.check_output("mount").split("\n")
-
-        points_to_umount = [line.split(" ")[2]
-                            for line in mount_lines
-                            if len(line) >= 3 and line.split(" ")[2].startswith(directory)]
-        ret = 0
-        for point in reversed(points_to_umount):
-            ret = subprocess.call(["umount", point])
-            if ret != 0:
-                ret = 1
-                logger.warning(m18n.n('backup_cleaning_failed', point))
-                continue
-
-        return ret
-
     def _check_is_enough_free_space(self):
         """
         Check free space in repository or output directory before to backup
@@ -1621,7 +1620,16 @@ class BackupMethod(object):
                     # 'NUMBER OF HARD LINKS > 1' see #1043
                     cron_path = os.path.abspath('/etc/cron') + '.'
                     if not os.path.abspath(src).startswith(cron_path):
+                        try:
                             os.link(src, dest)
+                        except Exception as e:
+                            # This kind of situation may happen when src and dest are on different
+                            # logical volume ... even though the st_dev check previously match...
+                            # E.g. this happens when running an encrypted hard drive
+                            # where everything is mapped to /dev/mapper/some-stuff
+                            # yet there are different devices behind it or idk ...
+                            logger.warning("Could not link %s to %s (%s) ... falling back to regular copy." % (src, dest, str(e)))
+                        else:
                             # Success, go to next file to organize
                             continue
 
@@ -1932,8 +1940,9 @@ class CustomBackupMethod(BackupMethod):
 
         ret = hook_callback('backup_method', [self.method],
                             args=self._get_args('need_mount'))
-
+        ret_succeed = [hook for hook, infos in ret.items()
-        self._need_mount = True if ret['succeed'] else False
+                       if any(result["state"] == "succeed" for result in infos.values())]
+        self._need_mount = True if ret_succeed else False
         return self._need_mount
 
     def backup(self):
@@ -1946,7 +1955,10 @@ class CustomBackupMethod(BackupMethod):
 
         ret = hook_callback('backup_method', [self.method],
                             args=self._get_args('backup'))
-        if ret['failed']:
+
+        ret_failed = [hook for hook, infos in ret.items()
+                      if any(result["state"] == "failed" for result in infos.values())]
+        if ret_failed:
             raise YunohostError('backup_custom_backup_error')
 
     def mount(self, restore_manager):
@@ -1959,7 +1971,10 @@ class CustomBackupMethod(BackupMethod):
         super(CustomBackupMethod, self).mount(restore_manager)
         ret = hook_callback('backup_method', [self.method],
                             args=self._get_args('mount'))
-        if ret['failed']:
+
+        ret_failed = [hook for hook, infos in ret.items()
+                      if any(result["state"] == "failed" for result in infos.values())]
+        if ret_failed:
             raise YunohostError('backup_custom_mount_error')
 
     def _get_args(self, action):
@@ -2011,6 +2026,7 @@ def backup_create(name=None, description=None, methods=[],
         # Check that output directory is empty
         if os.path.isdir(output_directory) and no_compress and \
                 os.listdir(output_directory):
+
             raise YunohostError('backup_output_directory_not_empty')
     elif no_compress:
         raise YunohostError('backup_output_directory_required')
@@ -2315,6 +2331,30 @@ def _call_for_each_path(self, callback, csv_path=None):
             callback(self, row['source'], row['dest'])
 
 
+def _recursive_umount(directory):
+    """
+    Recursively umount sub directories of a directory
+
+    Args:
+        directory -- a directory path
+    """
+    mount_lines = subprocess.check_output("mount").split("\n")
+
+    points_to_umount = [line.split(" ")[2]
+                        for line in mount_lines
+                        if len(line) >= 3 and line.split(" ")[2].startswith(directory)]
+
+    everything_went_fine = True
+    for point in reversed(points_to_umount):
+        ret = subprocess.call(["umount", point])
+        if ret != 0:
+            everything_went_fine = False
+            logger.warning(m18n.n('backup_cleaning_failed', point))
+            continue
+
+    return everything_went_fine
+
+
 def free_space_in_directory(dirpath):
     stat = os.statvfs(dirpath)
     return stat.f_frsize * stat.f_bavail

src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py

@@ -49,10 +49,6 @@ class MyMigration(Migration):
         if dsa:
             settings_set("service.ssh.allow_deprecated_dsa_hostkey", True)
 
-        # Create sshd_config.d dir
-        if not os.path.exists(SSHD_CONF + '.d'):
-            mkdir(SSHD_CONF + '.d', 0o755, uid='root', gid='root')
-
         # Here, we make it so that /etc/ssh/sshd_config is managed
         # by the regen conf (in particular in the case where the
         # from_script flag is present - in which case it was *not*

src/yunohost/dyndns.py

@@ -119,6 +119,9 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom
         subscribe_host -- Dynette HTTP API to subscribe to
 
     """
+    if len(glob.glob('/etc/yunohost/dyndns/*.key')) != 0 or os.path.exists('/etc/cron.d/yunohost-dyndns'):
+        raise YunohostError('domain_dyndns_already_subscribed')
+
     if domain is None:
         domain = _get_maindomain()
         operation_logger.related_to.append(('domain', domain))
@@ -144,7 +147,8 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom
                       'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain)
             os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private')
 
-        key_file = glob.glob('/etc/yunohost/dyndns/*.key')[0]
+        private_file = glob.glob('/etc/yunohost/dyndns/*%s*.private' % domain)[0]
+        key_file = glob.glob('/etc/yunohost/dyndns/*%s*.key' % domain)[0]
         with open(key_file) as f:
             key = f.readline().strip().split(' ', 6)[-1]
 
@@ -152,9 +156,13 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom
     # Send subscription
     try:
         r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30)
-    except requests.ConnectionError:
+    except Exception as e:
-        raise YunohostError('no_internet_connection')
+        os.system("rm -f %s" % private_file)
+        os.system("rm -f %s" % key_file)
+        raise YunohostError('dyndns_registration_failed', error=str(e))
     if r.status_code != 201:
+        os.system("rm -f %s" % private_file)
+        os.system("rm -f %s" % key_file)
         try:
             error = json.loads(r.text)['error']
         except:
@@ -333,7 +341,8 @@ def _guess_current_dyndns_domain(dyn_host):
     """
 
     # Retrieve the first registered domain
-    for path in glob.iglob('/etc/yunohost/dyndns/K*.private'):
+    paths = list(glob.iglob('/etc/yunohost/dyndns/K*.private'))
+    for path in paths:
         match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path)
         if not match:
             match = RE_DYNDNS_PRIVATE_KEY_SHA512.match(path)
@@ -343,7 +352,9 @@ def _guess_current_dyndns_domain(dyn_host):
 
         # Verify if domain is registered (i.e., if it's available, skip
         # current domain beause that's not the one we want to update..)
-        if _dyndns_available(dyn_host, _domain):
+        # If there's only 1 such key found, then avoid doing the request
+        # for nothing (that's very probably the one we want to find ...)
+        if len(paths) > 1 and _dyndns_available(dyn_host, _domain):
             continue
         else:
             return (_domain, path)

src/yunohost/hook.py

@@ -31,6 +31,7 @@ from glob import iglob
 from moulinette import m18n
 from yunohost.utils.error import YunohostError
 from moulinette.utils import log
+from moulinette.utils.filesystem import read_json
 
 HOOK_FOLDER = '/usr/share/yunohost/hooks/'
 CUSTOM_HOOK_FOLDER = '/etc/yunohost/hooks.d/'
@@ -228,7 +229,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
             (name, priority, path, succeed) as arguments
 
     """
-    result = {'succeed': {}, 'failed': {}}
+    result = {}
     hooks_dict = {}
 
     # Retrieve hooks
@@ -278,20 +279,20 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
             try:
                 hook_args = pre_callback(name=name, priority=priority,
                                          path=path, args=args)
-                hook_exec(path, args=hook_args, chdir=chdir, env=env,
+                hook_return = hook_exec(path, args=hook_args, chdir=chdir, env=env,
-                          no_trace=no_trace, raise_on_error=True)
+                          no_trace=no_trace, raise_on_error=True)[1]
             except YunohostError as e:
                 state = 'failed'
+                hook_return = {}
                 logger.error(e.strerror, exc_info=1)
                 post_callback(name=name, priority=priority, path=path,
                               succeed=False)
             else:
                 post_callback(name=name, priority=priority, path=path,
                               succeed=True)
-            try:
+            if not name in result:
-                result[state][name].append(path)
+                result[name] = {}
-            except KeyError:
+            result[name][path] = {'state' : state, 'stdreturn' : hook_return }
-                result[state][name] = [path]
     return result
 
 
@@ -339,6 +340,11 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
     stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo")
     env['YNH_STDINFO'] = stdinfo
 
+    stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn")
+    with open(stdreturn, 'w') as f:
+        f.write('')
+    env['YNH_STDRETURN'] = stdreturn
+
     # Construct command to execute
     if user == "root":
         command = ['sh', '-c']
@@ -385,10 +391,27 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
             raise YunohostError('hook_exec_not_terminated', path=path)
         else:
             logger.error(m18n.n('hook_exec_not_terminated', path=path))
-            return 1
+            return 1, {}
     elif raise_on_error and returncode != 0:
         raise YunohostError('hook_exec_failed', path=path)
-    return returncode
+
+    raw_content = None
+    try:
+        with open(stdreturn, 'r') as f:
+            raw_content = f.read()
+        if raw_content != '':
+            returnjson = read_json(stdreturn)
+        else:
+            returnjson = {}
+    except Exception as e:
+        raise YunohostError('hook_json_return_error', path=path, msg=str(e),
+                            raw_content=raw_content)
+    finally:
+        stdreturndir = os.path.split(stdreturn)[0]
+        os.remove(stdreturn)
+        os.rmdir(stdreturndir)
+
+    return returncode, returnjson
 
 
 def _extract_filename_parts(filename):

src/yunohost/service.py

@@ -493,12 +493,16 @@ def service_regen_conf(operation_logger, names=[], with_diff=False, force=False,
 
     pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call)
 
-    # Update the services name
+    # Keep only the hook names with at least one success
-    names = pre_result['succeed'].keys()
+    names = [hook for hook, infos in pre_result.items()
+             if any(result["state"] == "succeed" for result in infos.values())]
 
+    # FIXME : what do in case of partial success/failure ...
     if not names:
+        ret_failed = [hook for hook, infos in pre_result.items()
+                      if any(result["state"] == "failed" for result in infos.values())]
         raise YunohostError('service_regenconf_failed',
-                            services=', '.join(pre_result['failed']))
+                            services=', '.join(ret_failed))
 
     # Set the processing method
     _regen = _process_regen_conf if not dry_run else lambda *a, **k: True

src/yunohost/tests/test_backuprestore.py

@@ -10,7 +10,7 @@ from moulinette import m18n
 from moulinette.core import init_authenticator
 from yunohost.app import app_install, app_remove, app_ssowatconf
 from yunohost.app import _is_installed
-from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete
+from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete, _recursive_umount
 from yunohost.domain import _get_maindomain
 from yunohost.utils.error import YunohostError
 
@@ -42,7 +42,7 @@ def setup_function(function):
 
     assert len(backup_list()["archives"]) == 0
 
-    markers = function.__dict__.keys()
+    markers = [m.name for m in function.__dict__.get("pytestmark",[])]
 
     if "with_wordpress_archive_from_2p4" in markers:
         add_archive_wordpress_from_2p4()
@@ -82,7 +82,7 @@ def teardown_function(function):
     delete_all_backups()
     uninstall_test_apps_if_needed()
 
-    markers = function.__dict__.keys()
+    markers = [m.name for m in function.__dict__.get("pytestmark",[])]
 
     if "clean_opt_dir" in markers:
         shutil.rmtree("/opt/test_backup_output_directory")
@@ -571,7 +571,7 @@ def test_backup_binds_are_readonly(monkeypatch):
 
         assert "Read-only file system" in output
 
-        if self._recursive_umount(self.work_dir) > 0:
+        if not _recursive_umount(self.work_dir):
             raise Exception("Backup cleaning failed !")
 
         self.clean()