Merge branch 'stretch-unstable' into add_ynh_systemd_action

bin/yunohost-api

@@ -201,12 +201,10 @@ if __name__ == '__main__':
     _init_moulinette(opts.use_websocket, opts.debug, opts.verbose)
 
     # Run the server
-    from yunohost.utils.packages import ynh_packages_version
     ret = moulinette.api(
         _retrieve_namespaces(),
         host=opts.host, port=opts.port, routes={
             ('GET', '/installed'): is_installed,
-            ('GET', '/version'): ynh_packages_version,
         }, use_cache=opts.use_cache, use_websocket=opts.use_websocket
     )
     sys.exit(ret)

data/helpers.d/backend

@@ -121,7 +121,7 @@ ynh_add_systemd_config () {
 	# Manage arguments with getopts
 	ynh_handle_getopts_args "$@"
 	local service="${service:-$app}"
-	local template="${nonappend:-systemd.service}"
+	local template="${template:-systemd.service}"
 
 	finalsystemdconf="/etc/systemd/system/$service.service"
 	ynh_backup_if_checksum_is_different --file="$finalsystemdconf"
@@ -288,3 +288,149 @@ ynh_remove_fpm_config () {
 	ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1
 	ynh_systemd_action --service_name=$fpm_service --action=reload
 }
+
+# Create a dedicated fail2ban config (jail and filter conf files)
+#
+# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports]
+# | arg: -l, --logpath=   - Log file to be checked by fail2ban
+# | arg: -r, --failregex= - Failregex to be looked for by fail2ban
+# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3
+# | arg: -p, --ports=     - Ports blocked for a banned IP address - default: http,https
+#
+# -----------------------------------------------------------------------------
+#
+# usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"]
+# | arg: -t, --use_template - Use this helper in template mode
+# | arg: -v, --others_var=  - List of others variables to replace separeted by a space
+# |                           for example : 'var_1 var_2 ...'
+#
+# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf
+#   __APP__      by  $app
+#
+#  You can dynamically replace others variables by example :
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+#
+# Generally your template will look like that by example (for synapse):
+#
+# f2b_jail.conf:
+#     [__APP__]
+#     enabled = true
+#     port = http,https
+#     filter = __APP__
+#     logpath = /var/log/__APP__/logfile.log
+#     maxretry = 3
+#
+# f2b_filter.conf:
+#     [INCLUDES]
+#     before = common.conf
+#     [Definition]
+#
+#     # Part of regex definition (just used to make more easy to make the global regex)
+#     __synapse_start_line = .? \- synapse\..+ \-
+#
+#    # Regex definition.
+#    failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- <HOST> \- \d+ \- Received request\: POST /_matrix/client/r0/login\??<SKIPLINES>%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'<SKIPLINES>%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$
+#
+#     ignoreregex =
+#
+# -----------------------------------------------------------------------------
+#
+# Note about the "failregex" option:
+#          regex to match the password failure messages in the logfile. The
+#          host must be matched by a group named "host". The tag "<HOST>" can
+#          be used for standard IP/hostname matching and is only an alias for
+#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
+#
+#          You can find some more explainations about how to make a regex here :
+#          https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters
+#
+# Note that the logfile need to exist before to call this helper !!
+#
+# To validate your regex you can test with this command:
+# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf
+#
+ynh_add_fail2ban_config () {
+  # Declare an array to define the options of this helper.
+  declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=)
+  local logpath
+  local failregex
+  local max_retry
+  local ports
+  local others_var
+  local use_template
+  # Manage arguments with getopts
+  ynh_handle_getopts_args "$@"
+  use_template="${use_template:-0}"
+  max_retry=${max_retry:-3}
+  ports=${ports:-http,https}
+
+  finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf"
+  finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf"
+  ynh_backup_if_checksum_is_different "$finalfail2banjailconf"
+  ynh_backup_if_checksum_is_different "$finalfail2banfilterconf"
+
+  if [ $use_template -eq 1 ]
+  then
+    # Usage 2, templates
+    cp ../conf/f2b_jail.conf $finalfail2banjailconf
+    cp ../conf/f2b_filter.conf $finalfail2banfilterconf
+
+    if [ -n "${app:-}" ]
+    then
+      ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf"
+      ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf"
+    fi
+
+    # Replace all other variable given as arguments
+    for var_to_replace in ${others_var:-}; do
+      # ${var_to_replace^^} make the content of the variable on upper-cases
+      # ${!var_to_replace} get the content of the variable named $var_to_replace
+      ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf"
+      ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf"
+    done
+
+  else
+    # Usage 1, no template. Build a config file from scratch.
+    test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing."
+    test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing."
+
+    tee $finalfail2banjailconf <<EOF
+[$app]
+enabled = true
+port = $ports
+filter = $app
+logpath = $logpath
+maxretry = $max_retry
+EOF
+
+    tee $finalfail2banfilterconf <<EOF
+[INCLUDES]
+before = common.conf
+[Definition]
+failregex = $failregex
+ignoreregex =
+EOF
+  fi
+
+  # Common to usage 1 and 2.
+  ynh_store_file_checksum "$finalfail2banjailconf"
+  ynh_store_file_checksum "$finalfail2banfilterconf"
+
+  systemctl try-reload-or-restart fail2ban
+
+  local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")"
+  if [[ -n "$fail2ban_error" ]]; then
+    ynh_print_err --message="Fail2ban failed to load the jail for $app"
+    ynh_print_warn --message="${fail2ban_error#*WARNING}"
+  fi
+}
+
+# Remove the dedicated fail2ban config (jail and filter conf files)
+#
+# usage: ynh_remove_fail2ban_config
+ynh_remove_fail2ban_config () {
+  ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf"
+  ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf"
+  systemctl try-reload-or-restart fail2ban
+}

data/helpers.d/debug

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Debugger for app packagers
+#
+# usage: ynh_debug [--message=message] [--trace=1/0]
+# | arg: -m, --message= - The text to print
+# | arg: -t, --trace=   - Turn on or off the trace of the script. Usefull to trace nonly a small part of a script.
+ynh_debug () {
+	# Disable set xtrace for the helper itself, to not pollute the debug log
+	set +x
+	# Declare an array to define the options of this helper.
+	local legacy_args=mt
+	declare -Ar args_array=( [m]=message= [t]=trace= )
+	local message
+	local trace
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	# Redisable xtrace, ynh_handle_getopts_args set it back
+	set +x
+	message=${message:-}
+	trace=${trace:-}
+
+	if [ -n "$message" ]
+	then
+		ynh_print_log "\e[34m\e[1m[DEBUG]\e[0m ${message}" >&2
+	fi
+
+	if [ "$trace" == "1" ]
+	then
+		ynh_debug --message="Enable debugging"
+		set +x
+		# Get the current file descriptor of xtrace
+		old_bash_xtracefd=$BASH_XTRACEFD
+		# Add the current file name and the line number of any command currently running while tracing.
+		PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: '
+		# Force xtrace to stderr
+		BASH_XTRACEFD=2
+	fi
+	if [ "$trace" == "0" ]
+	then
+		ynh_debug --message="Disable debugging"
+		set +x
+		# Put xtrace back to its original fild descriptor
+		BASH_XTRACEFD=$old_bash_xtracefd
+	fi
+	# Renable set xtrace
+	set -x
+}
+
+# Execute a command and print the result as debug
+#
+# usage: ynh_debug_exec command to execute
+# usage: ynh_debug_exec "command to execute | following command"
+# In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe.
+#
+# | arg: command - command to execute
+ynh_debug_exec () {
+	ynh_debug --message="$(eval $@)"
+}

data/helpers.d/getopts

@@ -150,6 +150,9 @@ ynh_handle_getopts_args () {
 									# If there's already another value for this option, add a ; before adding the new value
 									eval ${option_var}+="\;"
 								fi
+								# Escape double quote to prevent any interpretation during the eval
+								all_args[$i]="${all_args[$i]//\"/\\\"}"
+
 								eval ${option_var}+=\"${all_args[$i]}\"
 								shift_value=$(( shift_value + 1 ))
 							fi
@@ -188,6 +191,9 @@ ynh_handle_getopts_args () {
 				# The variable name will be stored in 'option_var'
 				local option_var="${args_array[$option_flag]%=}"
 
+				# Escape double quote to prevent any interpretation during the eval
+				arguments[$i]="${arguments[$i]//\"/\\\"}"
+
 				# Store each value given as argument in the corresponding variable
 				# The values will be stored in the same order than $args_array
 				eval ${option_var}+=\"${arguments[$i]}\"

data/helpers.d/package

@@ -17,6 +17,21 @@ ynh_wait_dpkg_free() {
             # Sleep an exponential time at each round
             sleep $(( try * try ))
         else
+            # Check if dpkg hasn't been interrupted and is fully available.
+            # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174
+            local dpkg_dir="/var/lib/dpkg/updates/"
+
+            # For each file in $dpkg_dir
+            while read dpkg_file <&9
+            do
+                # Check if the name of this file contains only numbers.
+                if echo "$dpkg_file" | grep -Pq "^[[:digit:]]*$"
+                then
+                    # If so, that a remaining of dpkg.
+                    ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem."
+                    return 1
+                fi
+            done 9<<< "$(ls -1 $dpkg_dir)"
             return 0
         fi
     done
@@ -71,7 +86,7 @@ ynh_package_version() {
 # usage: ynh_apt update
 ynh_apt() {
     ynh_wait_dpkg_free
-    DEBIAN_FRONTEND=noninteractive sudo apt-get -y $@
+    DEBIAN_FRONTEND=noninteractive apt-get -y $@
 }
 
 # Update package index files

data/helpers.d/print

@@ -155,3 +155,81 @@ ynh_print_ON () {
 	# Print an echo only for the log, to be able to know that ynh_print_ON has been called.
 	echo ynh_print_ON > /dev/null
 }
+
+# Print a message as INFO and show progression during an app script
+#
+# usage: ynh_script_progression --message=message [--weight=weight] [--time]
+# | arg: -m, --message= - The text to print
+# | arg: -w, --weight=  - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script.
+# | arg: -t, --time=    - Print the execution time since the last call to this helper. Especially usefull to define weights.
+# | arg: -l, --last=    - Use for the last call of the helper, to fill te progression bar.
+increment_progression=0
+previous_weight=0
+# Define base_time when the file is sourced
+base_time=$(date +%s)
+ynh_script_progression () {
+	# Declare an array to define the options of this helper.
+	declare -Ar args_array=( [m]=message= [w]=weight= [t]=time [l]=last )
+	local message
+	local weight
+	local time
+	local last
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+	weight=${weight:-1}
+	time=${time:-0}
+	last=${last:-0}
+
+	# Get execution time since the last $base_time
+	local exec_time=$(( $(date +%s) - $base_time ))
+	base_time=$(date +%s)
+
+	# Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented.
+	local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)"
+	# Get the number of call with a weight value
+	local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0)
+
+	# Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight
+	local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]].*\)/\1/g')"
+	# Get the weight of each occurrences of 'ynh_script_progression' in the script using -w
+	local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]].*\)/\1/g')"
+	# Each value will be on a different line.
+	# Remove each 'end of line' and replace it by a '+' to sum the values.
+	local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 ))
+
+	# max_progression is a total number of calls to this helper.
+	#    Less the number of calls with a weight value.
+	#    Plus the total of weight values
+	local max_progression=$(( $helper_calls - $weight_calls + $weight_values ))
+
+	# Increment each execution of ynh_script_progression in this script by the weight of the previous call.
+	increment_progression=$(( $increment_progression + $previous_weight ))
+	# Store the weight of the current call in $previous_weight for next call
+	previous_weight=$weight
+
+	# Set the scale of the progression bar
+	local scale=20
+	# progress_string(1,2) should have the size of the scale.
+	local progress_string1="####################"
+	local progress_string0="...................."
+
+	# Reduce $increment_progression to the size of the scale
+	if [ $last -eq 0 ]
+	then
+		local effective_progression=$(( $increment_progression * $scale / $max_progression ))
+	# If last is specified, fill immediately the progression_bar
+	else
+		local effective_progression=$scale
+	fi
+
+	# Build $progression_bar from progress_string(1,2) according to $effective_progression
+	local progression_bar="${progress_string1:0:$effective_progression}${progress_string0:0:$(( $scale - $effective_progression ))}"
+
+	local print_exec_time=""
+	if [ $time -eq 1 ]
+	then
+		print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]"
+	fi
+
+	ynh_print_info "[$progression_bar] > ${message}${print_exec_time}"
+}

data/helpers.d/system

@@ -148,4 +148,107 @@ ynh_clean_check_starting () {
 	# Stop the execution of tail.
 	kill -s 15 $pid_tail 2>&1
 	ynh_secure_remove "$templog" 2>&1
+
+# Read the value of a key in a ynh manifest file
+#
+# usage: ynh_read_manifest manifest key
+# | arg: -m, --manifest= - Path of the manifest to read
+# | arg: -k, --key= - Name of the key to find
+ynh_read_manifest () {
+	# Declare an array to define the options of this helper.
+        declare -Ar args_array=( [m]=manifest= [k]=manifest_key= )
+        local manifest
+        local manifest_key
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	if [ ! -e "$manifest" ]; then
+		# If the manifest isn't found, try the common place for backup and restore script.
+		manifest="../settings/manifest.json"
+	fi
+
+	jq ".$manifest_key" "$manifest" --raw-output
+}
+
+# Read the upstream version from the manifest
+# The version number in the manifest is defined by <upstreamversion>~ynh<packageversion>
+# For example : 4.3-2~ynh3
+# This include the number before ~ynh
+# In the last example it return 4.3-2
+#
+# usage: ynh_app_upstream_version [-m manifest]
+# | arg: -m, --manifest= - Path of the manifest to read
+ynh_app_upstream_version () {
+    declare -Ar args_array=( [m]=manifest= )
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    manifest="${manifest:-../manifest.json}"
+    version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version")
+    echo "${version_key/~ynh*/}"
+}
+
+# Read package version from the manifest
+# The version number in the manifest is defined by <upstreamversion>~ynh<packageversion>
+# For example : 4.3-2~ynh3
+# This include the number after ~ynh
+# In the last example it return 3
+#
+# usage: ynh_app_package_version [-m manifest]
+# | arg: -m, --manifest= - Path of the manifest to read
+ynh_app_package_version () {
+    declare -Ar args_array=( [m]=manifest= )
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    manifest="${manifest:-../manifest.json}"
+    version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version")
+    echo "${version_key/*~ynh/}"
+}
+
+# Checks the app version to upgrade with the existing app version and returns:
+# - UPGRADE_APP if the upstream app version has changed
+# - UPGRADE_PACKAGE if only the YunoHost package has changed
+#
+## It stops the current script without error if the package is up-to-date
+#
+# This helper should be used to avoid an upgrade of an app, or the upstream part
+# of it, when it's not needed
+#
+# To force an upgrade, even if the package is up to date,
+# you have to set the variable YNH_FORCE_UPGRADE before.
+# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp
+#
+# usage: ynh_check_app_version_changed
+ynh_check_app_version_changed () {
+  local force_upgrade=${YNH_FORCE_UPGRADE:-0}
+  local package_check=${PACKAGE_CHECK_EXEC:-0}
+
+  # By default, upstream app version has changed
+  local return_value="UPGRADE_APP"
+
+  local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0)
+  local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")"
+  local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0)
+  local update_upstream_version="$(ynh_app_upstream_version)"
+
+  if [ "$current_version" == "$update_version" ] ; then
+      # Complete versions are the same
+      if [ "$force_upgrade" != "0" ]
+      then
+        echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2
+        unset YNH_FORCE_UPGRADE
+      elif [ "$package_check" != "0" ]
+      then
+        echo "Upgrade forced for package check." >&2
+      else
+        ynh_die "Up-to-date, nothing to do" 0
+      fi
+  elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then
+    # Upstream versions are the same, only YunoHost package versions differ
+    return_value="UPGRADE_PACKAGE"
+  fi
+  echo $return_value
 }

data/helpers.d/utils

@@ -257,7 +257,14 @@ ynh_setup_source () {
 # | arg: ...         - (Optionnal) More POST keys and values
 ynh_local_curl () {
 	# Define url of page to curl
-	local full_page_url=https://localhost$path_url$1
+	local local_page=$(ynh_normalize_url_path $1)
+	local full_path=$path_url$local_page
+	
+	if [ "${path_url}" == "/" ]; then 
+		full_path=$local_page
+	fi
+	
+	local full_page_url=https://localhost$full_path
 
 	# Concatenate all other arguments with '&' to prepare POST data
 	local POST_data=""

data/hooks/backup/05-conf_ldap

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ldap"

data/hooks/backup/08-conf_ssh

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ssh"

data/hooks/backup/11-conf_ynh_mysql

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh/mysql"

data/hooks/backup/14-conf_ssowat

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ssowat"

data/hooks/backup/17-data_home

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/data/home"

data/hooks/backup/20-conf_ynh_firewall

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh/firewall"

data/hooks/backup/21-conf_ynh_certs

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh/certs"

data/hooks/backup/23-data_mail

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/data/mail"

data/hooks/backup/26-conf_xmpp

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/xmpp"

data/hooks/backup/29-conf_nginx

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/nginx"

data/hooks/backup/32-conf_cron

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/cron"

data/hooks/backup/40-conf_ynh_currenthost

@@ -4,7 +4,7 @@
 set -eu
 
 # Source YNH helpers
-source /usr/share/yunohost/helpers.d/filesystem
+source /usr/share/yunohost/helpers
 
 # Backup destination
 backup_dir="${1}/conf/ynh"

data/hooks/conf_regen/03-ssh

@@ -2,7 +2,7 @@
 
 set -e
 
-. /usr/share/yunohost/helpers.d/utils
+. /usr/share/yunohost/helpers
 
 do_pre_regen() {
     pending_dir=$1

data/hooks/conf_regen/15-nginx

@@ -2,7 +2,7 @@
 
 set -e
 
-. /usr/share/yunohost/helpers.d/utils
+. /usr/share/yunohost/helpers
 
 do_init_regen() {
   if [[ $EUID -ne 0 ]]; then

data/hooks/conf_regen/34-mysql

@@ -2,6 +2,7 @@
 
 set -e
 MYSQL_PKG="mariadb-server-10.1"
+. /usr/share/yunohost/helpers
 
 do_pre_regen() {
   pending_dir=$1
@@ -15,7 +16,6 @@ do_post_regen() {
   regen_conf_files=$1
 
   if [ ! -f /etc/yunohost/mysql ]; then
-      . /usr/share/yunohost/helpers.d/string
 
       # ensure that mysql is running
       sudo systemctl -q is-active mysql.service \
@@ -25,8 +25,6 @@ do_post_regen() {
       mysql_password=$(ynh_string_random 10)
       sudo mysqladmin -s -u root -pyunohost password "$mysql_password" || {
         if [ $FORCE -eq 1 ]; then
-            . /usr/share/yunohost/helpers.d/package
-
             echo "It seems that you have already configured MySQL." \
               "YunoHost needs to have a root access to MySQL to runs its" \
               "applications, and is going to reset the MySQL root password." \

data/hooks/conf_regen/43-dnsmasq

@@ -1,13 +1,11 @@
 #!/bin/bash
 
 set -e
+. /usr/share/yunohost/helpers
 
 do_pre_regen() {
   pending_dir=$1
 
-  # source ip helpers
-  . /usr/share/yunohost/helpers.d/ip
-
   cd /usr/share/yunohost/templates/dnsmasq
 
   # create directory for pending conf

data/hooks/restore/11-conf_ynh_mysql

@@ -1,6 +1,8 @@
 backup_dir="$1/conf/ynh/mysql"
 MYSQL_PKG="mariadb-server-10.1"
 
+. /usr/share/yunohost/helpers
+
 # ensure that mysql is running
 service mysql status >/dev/null 2>&1 \
   || service mysql start
@@ -11,13 +13,11 @@ service mysql status >/dev/null 2>&1 \
 new_pwd=$(sudo cat "${backup_dir}/root_pwd" || sudo cat "${backup_dir}/mysql")
 [ -z "$curr_pwd" ] && curr_pwd="yunohost"
 [ -z "$new_pwd" ] && {
-    . /usr/share/yunohost/helpers.d/string
     new_pwd=$(ynh_string_random 10)
 }
 
 # attempt to change it
 sudo mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || {
-  . /usr/share/yunohost/helpers.d/package
 
   echo "It seems that you have already configured MySQL." \
     "YunoHost needs to have a root access to MySQL to runs its" \

data/templates/nginx/plain/yunohost_admin.conf

@@ -60,7 +60,8 @@ server {
         if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) {
             return 403;
         }
-
+        # X-Robots-Tag to precise the rules applied.
+        add_header  X-Robots-Tag "nofollow, noindex, noarchive, nosnippet";
         # Redirect most of 404 to maindomain.tld/yunohost/sso
         access_by_lua_file /usr/share/ssowat/access.lua;
     }

debian/changelog

@@ -1,3 +1,9 @@
+yunohost (3.4.2.4) stable; urgency=low
+
+  - [fix] Meltdown vulnerability checker something outputing trash instead of pure json
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Tue, 19 Feb 2019 19:11:38  +0000
+
 yunohost (3.4.2.3) stable; urgency=low
 
   - [fix] Admin password appearing in logs after logging in on webadmin

locales/en.json

@@ -75,7 +75,7 @@
     "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'",
     "backup_archive_open_failed": "Unable to open the backup archive",
     "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup",
-    "backup_archive_writing_error": "Unable to add files to backup into the compressed archive",
+    "backup_archive_writing_error": "Unable to add files '{source:s}' (named in the archive: '{dest:s}') to backup into the compressed archive '{archive:s}'",
     "backup_ask_for_copying_if_needed": "Some files couldn't be prepared to be backuped using the method that avoid to temporarily waste space on the system. To perform the backup, {size:s}MB should be used temporarily. Do you agree?",
     "backup_borg_not_implemented": "Borg backup method is not yet implemented",
     "backup_cant_mount_uncompress_archive": "Unable to mount in readonly mode the uncompress archive directory",

src/yunohost/app.py

@@ -97,6 +97,9 @@ def app_fetchlist(url=None, name=None):
         name -- Name of the list
         url -- URL of remote JSON list
     """
+    if not url.endswith(".json"):
+        raise YunohostError("This is not a valid application list url. It should end with .json.")
+
     # If needed, create folder where actual appslists are stored
     if not os.path.exists(REPO_PATH):
         os.makedirs(REPO_PATH)
@@ -2345,6 +2348,7 @@ def _parse_app_instance_name(app_instance_name):
     True
     """
     match = re_app_instance_name.match(app_instance_name)
+    assert match, "Could not parse app instance name : %s" % app_instance_name
     appid = match.groupdict().get('appid')
     app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1
     return (appid, app_instance_nb)

src/yunohost/backup.py

@@ -1802,10 +1802,11 @@ class TarBackupMethod(BackupMethod):
                 # Add the "source" into the archive and transform the path into
                 # "dest"
                 tar.add(path['source'], arcname=path['dest'])
-            tar.close()
         except IOError:
-            logger.error(m18n.n('backup_archive_writing_error'), exc_info=1)
+            logger.error(m18n.n('backup_archive_writing_error', source=path['source'], archive=self._archive_file, dest=path['dest']), exc_info=1)
             raise YunohostError('backup_creation_failed')
+        finally:
+            tar.close()
 
         # Move info file
         shutil.copy(os.path.join(self.work_dir, 'info.json'),

src/yunohost/firewall.py

@@ -195,6 +195,7 @@ def firewall_reload(skip_upnp=False):
 
     """
     from yunohost.hook import hook_callback
+    from yunohost.service import _run_service_command
 
     reloaded = False
     errors = False
@@ -276,8 +277,7 @@ def firewall_reload(skip_upnp=False):
         # Refresh port forwarding with UPnP
         firewall_upnp(no_refresh=False)
 
-    # TODO: Use service_restart
-    os.system("service fail2ban restart")
+    _run_service_command("reload", "fail2ban")
 
     if errors:
         logger.warning(m18n.n('firewall_rules_cmd_failed'))

src/yunohost/tools.py

@@ -735,6 +735,14 @@ def _check_if_vulnerable_to_meltdown():
         output, err = call.communicate()
         assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
 
+        # If there are multiple lines, sounds like there was some messages
+        # in stdout that are not json >.> ... Try to get the actual json
+        # stuff which should be the last line
+        output = output.strip()
+        if "\n" in output:
+            logger.debug("Original meltdown checker output : %s" % output)
+            output = output.split("\n")[-1]
+
         CVEs = json.loads(output)
         assert len(CVEs) == 1
         assert CVEs[0]["NAME"] == "MELTDOWN"