From 92bf413d684d8c3aea0e00833a515a82f7070e2e Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 31 Mar 2017 10:02:40 +0200 Subject: [PATCH 01/39] [fix] Remove version from api --- bin/yunohost-api | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/yunohost-api b/bin/yunohost-api index d2b219f8b..054d5df84 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -192,12 +192,10 @@ if __name__ == '__main__': _init_moulinette(opts.use_websocket, opts.debug, opts.verbose) # Run the server - from yunohost.utils.packages import ynh_packages_version ret = moulinette.api( _retrieve_namespaces(), host=opts.host, port=opts.port, routes={ ('GET', '/installed'): is_installed, - ('GET', '/version'): ynh_packages_version, }, use_cache=opts.use_cache, use_websocket=opts.use_websocket ) sys.exit(ret) From 35d4a1001fcd04207a9341389b5614e55919229d Mon Sep 17 00:00:00 2001 From: Jimmy Monin Date: Sat, 2 Sep 2017 18:27:25 +0200 Subject: [PATCH 02/39] Add fail2ban helpers --- data/helpers.d/backend | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index c54e82754..0795be38e 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -183,3 +183,57 @@ ynh_remove_fpm_config () { ynh_secure_remove "/etc/php5/fpm/conf.d/20-$app.ini" 2>&1 sudo systemctl reload php5-fpm } + +# Create a dedicated fail2ban config (jail and filter conf files) +# +# usage: ynh_add_fail2ban_config log_file filter [max_retry [ports]] +# | arg: log_file - Log file to be checked by fail2ban +# | arg: failregex - Failregex to be looked for by fail2ban +# | arg: max_retry - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: ports - Ports blocked for a banned IP address - default: http,https +ynh_add_fail2ban_config () { + # Process parameters + logpath=$1 + failregex=$2 + max_retry=${3:-3} + ports=${4:-http,https} + + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" + finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" + ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 + ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 + + sudo tee $finalfail2banjailconf < Date: Sat, 19 May 2018 10:11:16 +0200 Subject: [PATCH 03/39] Update to latest version from Experimental_helpers --- data/helpers.d/backend | 44 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 0795be38e..e14095102 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -192,41 +192,47 @@ ynh_remove_fpm_config () { # | arg: max_retry - Maximum number of retries allowed before banning IP address - default: 3 # | arg: ports - Ports blocked for a banned IP address - default: http,https ynh_add_fail2ban_config () { - # Process parameters - logpath=$1 - failregex=$2 - max_retry=${3:-3} - ports=${4:-http,https} - - test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." - + # Process parameters + logpath=$1 + failregex=$2 + max_retry=${3:-3} + ports=${4:-http,https} + + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 - - sudo tee $finalfail2banjailconf <&2 + echo "WARNING${fail2ban_error#*WARNING}" >&2 + fi } # Remove the dedicated fail2ban config (jail and filter conf files) @@ -234,6 +240,6 @@ EOF # usage: ynh_remove_fail2ban_config ynh_remove_fail2ban_config () { ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" - ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" - sudo systemctl restart fail2ban -} \ No newline at end of file + ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" + systemctl reload fail2ban +} From 7752bc0fb7eab0811926c891be5bb9703cd3b23a Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 27 Aug 2018 21:51:36 +0200 Subject: [PATCH 04/39] Update fail2ban helpers from experimental helpers. --- data/helpers.d/backend | 73 +++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index e14095102..4e939331e 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -187,26 +187,31 @@ ynh_remove_fpm_config () { # Create a dedicated fail2ban config (jail and filter conf files) # # usage: ynh_add_fail2ban_config log_file filter [max_retry [ports]] -# | arg: log_file - Log file to be checked by fail2ban -# | arg: failregex - Failregex to be looked for by fail2ban -# | arg: max_retry - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: ports - Ports blocked for a banned IP address - default: http,https +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https ynh_add_fail2ban_config () { - # Process parameters - logpath=$1 - failregex=$2 - max_retry=${3:-3} - ports=${4:-http,https} - - test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + # Declare an array to define the options of this helper. + declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= ) + local logpath + local failregex + local max_retry + local ports + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + max_retry=${max_retry:-3} + ports=${ports:-http,https} - finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" - finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" - ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 - ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." - tee $finalfail2banjailconf <&2 - echo "WARNING${fail2ban_error#*WARNING}" >&2 - fi + if [ "$(lsb_release --codename --short)" != "jessie" ]; then + systemctl reload fail2ban + else + systemctl restart fail2ban + fi + local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")" + if [ -n "$fail2ban_error" ] + then + echo "[ERR] Fail2ban failed to load the jail for $app" >&2 + echo "WARNING${fail2ban_error#*WARNING}" >&2 + fi } # Remove the dedicated fail2ban config (jail and filter conf files) # # usage: ynh_remove_fail2ban_config ynh_remove_fail2ban_config () { - ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" - ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" - systemctl reload fail2ban + ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" + ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" + if [ "$(lsb_release --codename --short)" != "jessie" ]; then + systemctl reload fail2ban + else + systemctl restart fail2ban + fi } From 0fd9e179f16a92af63292e9e8b048623fda5ac2e Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 28 Aug 2018 00:14:37 +0200 Subject: [PATCH 05/39] Add ynh_check_app_version_changed - Add the new helper ynh_check_app_version_changed to check the version before an upgrade - Add also the helpers ynh_read_manifest, ynh_app_upstream_version and ynh_app_package_version. - These previous helper have been modified (from the experimental version) to support getopts - ynh_check_app_version_changed has been modified to use ynh_app_upstream_version --- data/helpers.d/system | 105 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/data/helpers.d/system b/data/helpers.d/system index 70cc57493..63b90ef13 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -53,3 +53,108 @@ ynh_abort_if_errors () { ynh_get_debian_release () { echo $(lsb_release --codename --short) } + +# Read the value of a key in a ynh manifest file +# +# usage: ynh_read_manifest manifest key +# | arg: -m, --manifest= - Path of the manifest to read +# | arg: -k, --key= - Name of the key to find +ynh_read_manifest () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=manifest= [k]=manifest_key= ) + local manifest + local manifest_key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + python3 -c "import sys, json;print(json.load(open('$manifest', encoding='utf-8'))['$manifest_key'])" +} + +# Read the upstream version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number before ~ynh +# In the last example it return 4.3-2 +# +# usage: ynh_app_upstream_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +ynh_app_upstream_version () { + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + if [ ! -e "$manifest_path" ]; then + manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + fi + version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + echo "${version_key/~ynh*/}" +} + +# Read package version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number after ~ynh +# In the last example it return 3 +# +# usage: ynh_app_package_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +ynh_app_package_version () { + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + if [ ! -e "$manifest_path" ]; then + manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + fi + version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + echo "${version_key/*~ynh/}" +} + +# Checks the app version to upgrade with the existing app version and returns: +# - UPGRADE_APP if the upstream app version has changed +# - UPGRADE_PACKAGE if only the YunoHost package has changed +# +## It stops the current script without error if the package is up-to-date +# +# This helper should be used to avoid an upgrade of an app, or the upstream part +# of it, when it's not needed +# +# To force an upgrade, even if the package is up to date, +# you have to set the variable YNH_FORCE_UPGRADE before. +# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp + +# usage: ynh_check_app_version_changed +ynh_check_app_version_changed () { + local force_upgrade=${YNH_FORCE_UPGRADE:-0} + local package_check=${PACKAGE_CHECK_EXEC:-0} + + # By default, upstream app version has changed + local return_value="UPGRADE_APP" + + local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0) + local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")" + local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0) + local update_upstream_version="$(ynh_app_upstream_version)" + + if [ "$current_version" == "$update_version" ] ; then + # Complete versions are the same + if [ "$force_upgrade" != "0" ] + then + echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2 + unset YNH_FORCE_UPGRADE + elif [ "$package_check" != "0" ] + then + echo "Upgrade forced for package check." >&2 + else + ynh_die "Up-to-date, nothing to do" 0 + fi + elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then + # Upstream versions are the same, only YunoHost package versions differ + return_value="UPGRADE_PACKAGE" + fi + echo $return_value +} From e0533a1a6a48bbce12fe8a961647f62436fa3649 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Aug 2018 00:29:24 +0200 Subject: [PATCH 06/39] Really important contribution to the PR :ninja: --- data/helpers.d/system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 63b90ef13..f09343953 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -126,7 +126,7 @@ ynh_app_package_version () { # To force an upgrade, even if the package is up to date, # you have to set the variable YNH_FORCE_UPGRADE before. # example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp - +# # usage: ynh_check_app_version_changed ynh_check_app_version_changed () { local force_upgrade=${YNH_FORCE_UPGRADE:-0} From 62c8f577c9f57e9164a7bbbaffd5a5a611113daa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 13:25:29 +0200 Subject: [PATCH 07/39] Remove old sudo --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 4e939331e..bc3ae6a7d 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -211,7 +211,7 @@ ynh_add_fail2ban_config () { ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 - sudo tee $finalfail2banjailconf < Date: Wed, 29 Aug 2018 13:27:25 +0200 Subject: [PATCH 08/39] Remove sudo --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index bc3ae6a7d..6b5ca1d37 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -220,7 +220,7 @@ logpath = $logpath maxretry = $max_retry EOF - sudo tee $finalfail2banfilterconf < Date: Sun, 13 Jan 2019 14:24:17 +0100 Subject: [PATCH 09/39] [fix] Replace manifest_path by manifest manifest_path isn't defined --- data/helpers.d/system | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index f09343953..a93b3ea6f 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -85,10 +85,10 @@ ynh_app_upstream_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest_path" ]; then - manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + if [ ! -e "$manifest" ]; then + manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place fi - version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/~ynh*/}" } @@ -107,10 +107,10 @@ ynh_app_package_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest_path" ]; then - manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + if [ ! -e "$manifest" ]; then + manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place fi - version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/*~ynh/}" } From 08869c329c5ac58645544f5672d53ab61927fa7b Mon Sep 17 00:00:00 2001 From: frju365 Date: Mon, 21 Jan 2019 21:17:33 +0100 Subject: [PATCH 10/39] Update yunohost_admin.conf --- data/templates/nginx/plain/yunohost_admin.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 3de66e3e6..06d1ef09c 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -68,7 +68,7 @@ server { if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { return 403; } - + add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; } From a42df2d8fe3371b5e1d5c0a7a325fa8524e7fe28 Mon Sep 17 00:00:00 2001 From: frju365 Date: Mon, 21 Jan 2019 21:39:07 +0100 Subject: [PATCH 11/39] Update yunohost_admin.conf --- data/templates/nginx/plain/yunohost_admin.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 06d1ef09c..917ad0d5e 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -65,9 +65,6 @@ server { location /yunohost { # Block crawlers bot - if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { - return 403; - } add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; From 7b7030544876178e606c389606766eff89da21c4 Mon Sep 17 00:00:00 2001 From: frju365 Date: Sat, 26 Jan 2019 23:51:22 +0100 Subject: [PATCH 12/39] Update yunohost_admin.conf --- data/templates/nginx/plain/yunohost_admin.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 917ad0d5e..24ea25072 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -65,6 +65,10 @@ server { location /yunohost { # Block crawlers bot + if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { + return 403; + } + # X-Robots-Tag to precise the rules applied. add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; From b9fa901bda11a97c9460ac5f426bc38a0e21c6c3 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 28 Jan 2019 16:31:54 +0100 Subject: [PATCH 13/39] Add ynh_script_progression --- data/helpers.d/print | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/data/helpers.d/print b/data/helpers.d/print index 2f451bc24..1d7f6f6fa 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -124,3 +124,81 @@ ynh_print_ON () { # Print an echo only for the log, to be able to know that ynh_print_ON has been called. echo ynh_print_ON > /dev/null } + +# Print a message as INFO and show progression during an app script +# +# usage: ynh_script_progression --message=message [--weight=weight] [--time] +# | arg: -m, --message= - The text to print +# | arg: -w, --weight= - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script. +# | arg: -t, --time= - Print the execution time since the last call to this helper. Especially usefull to define weights. +# | arg: -l, --last= - Use for the last call of the helper, to fill te progression bar. +increment_progression=0 +previous_weight=0 +# Define base_time when the file is sourced +base_time=$(date +%s) +ynh_script_progression () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=message= [w]=weight= [t]=time [l]=last ) + local message + local weight + local time + local last + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + weight=${weight:-1} + time=${time:-0} + last=${last:-0} + + # Get execution time since the last $base_time + local exec_time=$(( $(date +%s) - $base_time )) + base_time=$(date +%s) + + # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented. + local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)" + # Get the number of call with a weight value + local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0) + + # Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight + local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]].*\)/\1/g')" + # Get the weight of each occurrences of 'ynh_script_progression' in the script using -w + local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]].*\)/\1/g')" + # Each value will be on a different line. + # Remove each 'end of line' and replace it by a '+' to sum the values. + local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 )) + + # max_progression is a total number of calls to this helper. + # Less the number of calls with a weight value. + # Plus the total of weight values + local max_progression=$(( $helper_calls - $weight_calls + $weight_values )) + + # Increment each execution of ynh_script_progression in this script by the weight of the previous call. + increment_progression=$(( $increment_progression + $previous_weight )) + # Store the weight of the current call in $previous_weight for next call + previous_weight=$weight + + # Set the scale of the progression bar + local scale=20 + # progress_string(1,2) should have the size of the scale. + local progress_string1="####################" + local progress_string0="...................." + + # Reduce $increment_progression to the size of the scale + if [ $last -eq 0 ] + then + local effective_progression=$(( $increment_progression * $scale / $max_progression )) + # If last is specified, fill immediately the progression_bar + else + local effective_progression=$scale + fi + + # Build $progression_bar from progress_string(1,2) according to $effective_progression + local progression_bar="${progress_string1:0:$effective_progression}${progress_string0:0:$(( $scale - $effective_progression ))}" + + local print_exec_time="" + if [ $time -eq 1 ] + then + print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]" + fi + + ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" +} From d77bc92b6e4be77e30b2600ea099c1805f4a39fb Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Thu, 31 Jan 2019 17:54:25 +0100 Subject: [PATCH 14/39] Check if dpkg is broken --- data/helpers.d/package | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/data/helpers.d/package b/data/helpers.d/package index 2cbca4840..6bd655fe0 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -15,6 +15,21 @@ ynh_wait_dpkg_free() { # Sleep an exponential time at each round sleep $(( try * try )) else + # Check if dpkg hasn't been interrupted and is fully available. + # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + local dpkg_dir="/var/lib/dpkg/updates/" + + # For each file in $dpkg_dir + while read dpkg_file <&9 + do + # Check if the name of this file contains only numbers. + if echo "$dpkg_file" | grep -Pq "^[[:digit:]]*$" + then + # If so, that a remaining of dpkg. + ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + return 1 + fi + done 9<<< "$(ls -1 $dpkg_dir)" return 0 fi done From a742ca8939d1d136d1b998c96d7e4fb4046dcc3c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 9 Feb 2019 15:40:29 +0100 Subject: [PATCH 15/39] Attempt to get rid of annoying 'unable to initialize frontend' messages --- data/helpers.d/package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/package b/data/helpers.d/package index c0617abb2..1b7c38c5c 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -150,7 +150,7 @@ ynh_package_install_from_equivs () { cp "$controlfile" "${TMPDIR}/control" (cd "$TMPDIR" equivs-build ./control 1> /dev/null - dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) + DEBIAN_FRONTEND=noninteractive dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) ynh_package_install -f || ynh_die --message="Unable to install dependencies" [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR # Remove the temp dir. From 6ba12bb9ae77c60653969b6bc999d6be739f7685 Mon Sep 17 00:00:00 2001 From: Kayou Date: Sat, 9 Feb 2019 23:04:21 +0100 Subject: [PATCH 16/39] Fix ynh_local_curl --- data/helpers.d/utils | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 360174847..94ba8bd97 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -257,7 +257,15 @@ ynh_setup_source () { # | arg: ... - (Optionnal) More POST keys and values ynh_local_curl () { # Define url of page to curl - local full_page_url=https://localhost$path_url$1 + path_url=$(ynh_normalize_url_path $path_url) + local_page=$(ynh_normalize_url_path $1) + full_path=$path_url$local_page + + if [ "${path_url}" == "/" ]; then + full_path=$local_page + fi + + local full_page_url=https://localhost$full_path # Concatenate all other arguments with '&' to prepare POST data local POST_data="" From a0dfae269f2d1c6f570d55cdeef2d8a495a73a59 Mon Sep 17 00:00:00 2001 From: Kayou Date: Sat, 9 Feb 2019 23:13:48 +0100 Subject: [PATCH 17/39] Update utils --- data/helpers.d/utils | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 94ba8bd97..40bef7b88 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -258,8 +258,8 @@ ynh_setup_source () { ynh_local_curl () { # Define url of page to curl path_url=$(ynh_normalize_url_path $path_url) - local_page=$(ynh_normalize_url_path $1) - full_path=$path_url$local_page + local local_page=$(ynh_normalize_url_path $1) + local full_path=$path_url$local_page if [ "${path_url}" == "/" ]; then full_path=$local_page From 76e1c2e3955f599d51174a8bf8a68d64a301f8bf Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 10 Feb 2019 20:51:40 +0100 Subject: [PATCH 18/39] Fix template variable in ynh_add_systemd_config Merged as a micro decision, as it's only a small mistake. --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 5e6d564cf..c96315238 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -121,7 +121,7 @@ ynh_add_systemd_config () { # Manage arguments with getopts ynh_handle_getopts_args "$@" local service="${service:-$app}" - local template="${nonappend:-systemd.service}" + local template="${template:-systemd.service}" finalsystemdconf="/etc/systemd/system/$service.service" ynh_backup_if_checksum_is_different --file="$finalsystemdconf" From 573fab426834c575596e93c0f4ea7403fd47a4f4 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 10 Feb 2019 21:23:56 +0100 Subject: [PATCH 19/39] [fix] Escape double quote before eval in getopts Work in progress... Need to be tested. --- data/helpers.d/getopts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data/helpers.d/getopts b/data/helpers.d/getopts index efaa8d065..7055325f1 100644 --- a/data/helpers.d/getopts +++ b/data/helpers.d/getopts @@ -150,6 +150,9 @@ ynh_handle_getopts_args () { # If there's already another value for this option, add a ; before adding the new value eval ${option_var}+="\;" fi + # Escape double quote to prevent any interpretation during the eval + all_args[$i]="${all_args[$i]//\"/\\\"}" + eval ${option_var}+=\"${all_args[$i]}\" shift_value=$(( shift_value + 1 )) fi @@ -188,6 +191,9 @@ ynh_handle_getopts_args () { # The variable name will be stored in 'option_var' local option_var="${args_array[$option_flag]%=}" + # Escape double quote to prevent any interpretation during the eval + arguments[$i]="${arguments[$i]//\"/\\\"}" + # Store each value given as argument in the corresponding variable # The values will be stored in the same order than $args_array eval ${option_var}+=\"${arguments[$i]}\" From 3dbe9af7ddbaa082b22f385de13529384a6e0c19 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 10 Feb 2019 23:43:27 +0100 Subject: [PATCH 20/39] Create debug --- data/helpers.d/debug | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 data/helpers.d/debug diff --git a/data/helpers.d/debug b/data/helpers.d/debug new file mode 100644 index 000000000..a8b7c8d69 --- /dev/null +++ b/data/helpers.d/debug @@ -0,0 +1,59 @@ +#!/bin/bash + +# Debugger for app packagers +# +# usage: ynh_debug [--message=message] [--trace=1/0] +# | arg: -m, --message= - The text to print +# | arg: -t, --trace= - Turn on or off the trace of the script. Usefull to trace nonly a small part of a script. +ynh_debug () { + # Disable set xtrace for the helper itself, to not pollute the debug log + set +x + # Declare an array to define the options of this helper. + local legacy_args=mt + declare -Ar args_array=( [m]=message= [t]=trace= ) + local message + local trace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Redisable xtrace, ynh_handle_getopts_args set it back + set +x + message=${message:-} + trace=${trace:-} + + if [ -n "$message" ] + then + ynh_print_log "\e[34m\e[1m[DEBUG]\e[0m ${message}" >&2 + fi + + if [ "$trace" == "1" ] + then + ynh_debug --message="Enable debugging" + set +x + # Get the current file descriptor of xtrace + old_bash_xtracefd=$BASH_XTRACEFD + # Add the current file name and the line number of any command currently running while tracing. + PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: ' + # Force xtrace to stderr + BASH_XTRACEFD=2 + fi + if [ "$trace" == "0" ] + then + ynh_debug --message="Disable debugging" + set +x + # Put xtrace back to its original fild descriptor + BASH_XTRACEFD=$old_bash_xtracefd + fi + # Renable set xtrace + set -x +} + +# Execute a command and print the result as debug +# +# usage: ynh_debug_exec command to execute +# usage: ynh_debug_exec "command to execute | following command" +# In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# +# | arg: command - command to execute +ynh_debug_exec () { + ynh_debug --message="$(eval $@)" +} From b6c5b1fce933847a3a29e7e26c747792271b13b6 Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 11 Feb 2019 16:45:40 +0100 Subject: [PATCH 21/39] Don't normilize path_url --- data/helpers.d/utils | 1 - 1 file changed, 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 40bef7b88..5ba2946a2 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -257,7 +257,6 @@ ynh_setup_source () { # | arg: ... - (Optionnal) More POST keys and values ynh_local_curl () { # Define url of page to curl - path_url=$(ynh_normalize_url_path $path_url) local local_page=$(ynh_normalize_url_path $1) local full_path=$path_url$local_page From 2065b8914262d09bcaee92fb1c9fba7b4ab87ffa Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 12 Feb 2019 00:28:39 +0100 Subject: [PATCH 22/39] Use jq instead of python3 --- data/helpers.d/system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index a93b3ea6f..39af0092f 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -67,7 +67,7 @@ ynh_read_manifest () { # Manage arguments with getopts ynh_handle_getopts_args "$@" - python3 -c "import sys, json;print(json.load(open('$manifest', encoding='utf-8'))['$manifest_key'])" + jq ".$manifest_key" "$manifest" --raw-output } # Read the upstream version from the manifest From 140ae8e51a74e9db693b954ea22e08038ff1f5d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Tue, 12 Feb 2019 21:13:49 +0100 Subject: [PATCH 23/39] Change the way to manage the jail and filter file and improve documentations Using a template file make more easy to use a custom failregex. It also give the possiblitity to use custom settings in the fail2ban config --- data/helpers.d/backend | 116 ++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 47 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 6b5ca1d37..ac4d4ed94 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -186,59 +186,85 @@ ynh_remove_fpm_config () { # Create a dedicated fail2ban config (jail and filter conf files) # -# usage: ynh_add_fail2ban_config log_file filter [max_retry [ports]] -# | arg: -l, --logpath= - Log file to be checked by fail2ban -# | arg: -r, --failregex= - Failregex to be looked for by fail2ban -# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# usage: ynh_add_fail2ban_config "list of others variables to replace" +# +# | arg: list of others variables to replace separeted by a space +# | for example : 'var_1 var_2 ...' +# +# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf +# __APP__ by $app +# +# You can dynamically replace others variables by example : +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# +# Note about the "failregex" option: +# regex to match the password failure messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# Note that the logfile need to exist before to call this helper !! +# +# Generally your template will look like that by example (for synapse): +# +# f2b_jail.conf: +# [__APP__] +# enabled = true +# port = http,https +# filter = __APP__ +# logpath = /var/log/__APP__/logfile.log +# maxretry = 3 +# +# f2b_filter.conf: +# [INCLUDES] +# before = common.conf +# [Definition] +# +# # Part of regex definition (just used to make more easy to make the global regex) +# __synapse_start_line = .? \- synapse\..+ \- +# +# # Regex definition. +# failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ +# +# ignoreregex = +# +# To validate your regex you can test with this command: +# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf ynh_add_fail2ban_config () { - # Declare an array to define the options of this helper. - declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= ) - local logpath - local failregex - local max_retry - local ports - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - max_retry=${max_retry:-3} - ports=${ports:-http,https} - - test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + local others_var=${1:-} finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" - ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 - ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 + ynh_backup_if_checksum_is_different "$finalfail2banjailconf" + ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" - tee $finalfail2banjailconf <&2 echo "WARNING${fail2ban_error#*WARNING}" >&2 fi @@ -250,9 +276,5 @@ EOF ynh_remove_fail2ban_config () { ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" - if [ "$(lsb_release --codename --short)" != "jessie" ]; then - systemctl reload fail2ban - else - systemctl restart fail2ban - fi + systemctl try-reload-or-restart fail2ban } From e1ccab212a7241d1e76b955b6699c20c10bfc7c0 Mon Sep 17 00:00:00 2001 From: Josue-T Date: Wed, 13 Feb 2019 22:17:32 +0100 Subject: [PATCH 24/39] Reload fail2ban instead of restart --- src/yunohost/firewall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py index 1c44efe99..9d209dbb8 100644 --- a/src/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -195,6 +195,7 @@ def firewall_reload(skip_upnp=False): """ from yunohost.hook import hook_callback + from yunohost.service import _run_service_command reloaded = False errors = False @@ -276,8 +277,7 @@ def firewall_reload(skip_upnp=False): # Refresh port forwarding with UPnP firewall_upnp(no_refresh=False) - # TODO: Use service_restart - os.system("service fail2ban restart") + _run_service_command("reload", "fail2ban") if errors: logger.warning(m18n.n('firewall_rules_cmd_failed')) From bba393c45c5ee45a942f6c7be697e7a2415e14b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Fri, 15 Feb 2019 14:06:53 +0100 Subject: [PATCH 25/39] Add possibility to use template or predefined config --- data/helpers.d/backend | 70 +++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index ac4d4ed94..c16da8f95 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -185,11 +185,18 @@ ynh_remove_fpm_config () { } # Create a dedicated fail2ban config (jail and filter conf files) +# usage 1: ynh_add_fail2ban_config log_file filter [max_retry [ports]] +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https # -# usage: ynh_add_fail2ban_config "list of others variables to replace" +# ----------------------------------------------------------------------------- # -# | arg: list of others variables to replace separeted by a space -# | for example : 'var_1 var_2 ...' +# usage 2: ynh_add_fail2ban_config -t [-v "list of others variables to replace"] +# | arg: -t, --use_template - Use this helper in template mode +# | arg: -v, --others_var= - List of others variables to replace separeted by a space +# | for example : 'var_1 var_2 ...' # # This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf # __APP__ by $app @@ -235,7 +242,16 @@ ynh_remove_fpm_config () { # To validate your regex you can test with this command: # fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf ynh_add_fail2ban_config () { - local others_var=${1:-} + # Declare an array to define the options of this helper. + declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) + local logpath + local failregex + local max_retry + local ports + local others_var + local use_template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" @@ -245,18 +261,42 @@ ynh_add_fail2ban_config () { cp ../conf/f2b_jail.conf $finalfail2banjailconf cp ../conf/f2b_filter.conf $finalfail2banfilterconf - if test -n "${app:-}"; then - ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" - ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" - fi + if [[ ${use_template:-0} == 1 ]]; then + if test -n "${app:-}"; then + ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" + ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" + fi - # Replace all other variable given as arguments - for var_to_replace in $others_var; do - # ${var_to_replace^^} make the content of the variable on upper-cases - # ${!var_to_replace} get the content of the variable named $var_to_replace - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" - done + # Replace all other variable given as arguments + for var_to_replace in ${others_var:-}; do + # ${var_to_replace^^} make the content of the variable on upper-cases + # ${!var_to_replace} get the content of the variable named $var_to_replace + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" + done + else + max_retry=${max_retry:-3} + ports=${ports:-http,https} + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + tee $finalfail2banjailconf < Date: Fri, 15 Feb 2019 16:18:36 +0100 Subject: [PATCH 26/39] Update usage comment --- data/helpers.d/backend | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index c16da8f95..c0722a5ce 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -185,15 +185,15 @@ ynh_remove_fpm_config () { } # Create a dedicated fail2ban config (jail and filter conf files) -# usage 1: ynh_add_fail2ban_config log_file filter [max_retry [ports]] +# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] # | arg: -l, --logpath= - Log file to be checked by fail2ban # | arg: -r, --failregex= - Failregex to be looked for by fail2ban # | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https # # ----------------------------------------------------------------------------- # -# usage 2: ynh_add_fail2ban_config -t [-v "list of others variables to replace"] +# usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"] # | arg: -t, --use_template - Use this helper in template mode # | arg: -v, --others_var= - List of others variables to replace separeted by a space # | for example : 'var_1 var_2 ...' From 54eb3efec0dce598cab7cc21446cda8485af25dc Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 15 Feb 2019 16:21:30 +0100 Subject: [PATCH 27/39] Reorganize comment header --- data/helpers.d/backend | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index c0722a5ce..a8da55ef7 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -185,6 +185,7 @@ ynh_remove_fpm_config () { } # Create a dedicated fail2ban config (jail and filter conf files) +# # usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] # | arg: -l, --logpath= - Log file to be checked by fail2ban # | arg: -r, --failregex= - Failregex to be looked for by fail2ban @@ -205,17 +206,6 @@ ynh_remove_fpm_config () { # __VAR_1__ by $var_1 # __VAR_2__ by $var_2 # -# Note about the "failregex" option: -# regex to match the password failure messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) -# -# You can find some more explainations about how to make a regex here : -# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters -# -# Note that the logfile need to exist before to call this helper !! -# # Generally your template will look like that by example (for synapse): # # f2b_jail.conf: @@ -239,8 +229,22 @@ ynh_remove_fpm_config () { # # ignoreregex = # +# ----------------------------------------------------------------------------- +# +# Note about the "failregex" option: +# regex to match the password failure messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# Note that the logfile need to exist before to call this helper !! +# # To validate your regex you can test with this command: # fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf +# ynh_add_fail2ban_config () { # Declare an array to define the options of this helper. declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) From ccb0cad296c335b63f9691e234c19ef2a2e11211 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 15 Feb 2019 16:32:13 +0100 Subject: [PATCH 28/39] Fix various issues --- data/helpers.d/backend | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index a8da55ef7..a7d3a09a1 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -256,17 +256,23 @@ ynh_add_fail2ban_config () { local use_template # Manage arguments with getopts ynh_handle_getopts_args "$@" + use_template="${use_template:-0}" + max_retry=${max_retry:-3} + ports=${ports:-http,https} finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" ynh_backup_if_checksum_is_different "$finalfail2banjailconf" ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" - cp ../conf/f2b_jail.conf $finalfail2banjailconf - cp ../conf/f2b_filter.conf $finalfail2banfilterconf + if [ $use_template -eq 1 ] + then + # Usage 2, templates + cp ../conf/f2b_jail.conf $finalfail2banjailconf + cp ../conf/f2b_filter.conf $finalfail2banfilterconf - if [[ ${use_template:-0} == 1 ]]; then - if test -n "${app:-}"; then + if [ -n "${app:-}" ] + then ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" fi @@ -278,9 +284,9 @@ ynh_add_fail2ban_config () { ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" done + else - max_retry=${max_retry:-3} - ports=${ports:-http,https} + # Usage 1, no template. Build a config file from scratch. test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." @@ -302,6 +308,7 @@ ignoreregex = EOF fi + # Common to usage 1 and 2. ynh_store_file_checksum "$finalfail2banjailconf" ynh_store_file_checksum "$finalfail2banfilterconf" @@ -309,8 +316,8 @@ EOF local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then - echo "[ERR] Fail2ban failed to load the jail for $app" >&2 - echo "WARNING${fail2ban_error#*WARNING}" >&2 + ynh_print_err --message="Fail2ban failed to load the jail for $app" + ynh_print_warn --message="${fail2ban_error#*WARNING}" fi } From b62d72f77b31b9055e0a97aef68d3e2f19db30c9 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 16 Feb 2019 00:28:27 +0100 Subject: [PATCH 29/39] Factorize into ynh_read_manifest --- data/helpers.d/system | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 39af0092f..052ea5cec 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -67,6 +67,11 @@ ynh_read_manifest () { # Manage arguments with getopts ynh_handle_getopts_args "$@" + if [ ! -e "$manifest" ]; then + # If the manifest isn't found, try the common place for backup and restore script. + manifest="../settings/manifest.json" + fi + jq ".$manifest_key" "$manifest" --raw-output } @@ -85,9 +90,6 @@ ynh_app_upstream_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest" ]; then - manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place - fi version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/~ynh*/}" } @@ -107,9 +109,6 @@ ynh_app_package_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest" ]; then - manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place - fi version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/*~ynh/}" } From d495d315c0e1c9492e94e0e810a0e6cdb3d4c975 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 16:36:35 +0100 Subject: [PATCH 30/39] [fix] Loading only one helper file leads to errors because missing getopts --- data/hooks/backup/05-conf_ldap | 2 +- data/hooks/backup/08-conf_ssh | 2 +- data/hooks/backup/11-conf_ynh_mysql | 2 +- data/hooks/backup/14-conf_ssowat | 2 +- data/hooks/backup/17-data_home | 2 +- data/hooks/backup/20-conf_ynh_firewall | 2 +- data/hooks/backup/21-conf_ynh_certs | 2 +- data/hooks/backup/23-data_mail | 2 +- data/hooks/backup/26-conf_xmpp | 2 +- data/hooks/backup/29-conf_nginx | 2 +- data/hooks/backup/32-conf_cron | 2 +- data/hooks/backup/40-conf_ynh_currenthost | 2 +- data/hooks/conf_regen/03-ssh | 2 +- data/hooks/conf_regen/15-nginx | 2 +- data/hooks/conf_regen/34-mysql | 4 +--- data/hooks/conf_regen/43-dnsmasq | 4 +--- data/hooks/restore/11-conf_ynh_mysql | 4 ++-- 17 files changed, 18 insertions(+), 22 deletions(-) diff --git a/data/hooks/backup/05-conf_ldap b/data/hooks/backup/05-conf_ldap index b21103ede..9ae22095e 100755 --- a/data/hooks/backup/05-conf_ldap +++ b/data/hooks/backup/05-conf_ldap @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ldap" diff --git a/data/hooks/backup/08-conf_ssh b/data/hooks/backup/08-conf_ssh index ae422617e..ee976080c 100755 --- a/data/hooks/backup/08-conf_ssh +++ b/data/hooks/backup/08-conf_ssh @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ssh" diff --git a/data/hooks/backup/11-conf_ynh_mysql b/data/hooks/backup/11-conf_ynh_mysql index 60bd8c017..031707337 100755 --- a/data/hooks/backup/11-conf_ynh_mysql +++ b/data/hooks/backup/11-conf_ynh_mysql @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/mysql" diff --git a/data/hooks/backup/14-conf_ssowat b/data/hooks/backup/14-conf_ssowat index ca42d3369..d4db72493 100755 --- a/data/hooks/backup/14-conf_ssowat +++ b/data/hooks/backup/14-conf_ssowat @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ssowat" diff --git a/data/hooks/backup/17-data_home b/data/hooks/backup/17-data_home index f7a797b6b..af00d67e8 100755 --- a/data/hooks/backup/17-data_home +++ b/data/hooks/backup/17-data_home @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/home" diff --git a/data/hooks/backup/20-conf_ynh_firewall b/data/hooks/backup/20-conf_ynh_firewall index 4e08114e7..98be3eb09 100755 --- a/data/hooks/backup/20-conf_ynh_firewall +++ b/data/hooks/backup/20-conf_ynh_firewall @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/firewall" diff --git a/data/hooks/backup/21-conf_ynh_certs b/data/hooks/backup/21-conf_ynh_certs index f9687164d..a3912a995 100755 --- a/data/hooks/backup/21-conf_ynh_certs +++ b/data/hooks/backup/21-conf_ynh_certs @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/certs" diff --git a/data/hooks/backup/23-data_mail b/data/hooks/backup/23-data_mail index 618a0aafe..7fdc883fd 100755 --- a/data/hooks/backup/23-data_mail +++ b/data/hooks/backup/23-data_mail @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/mail" diff --git a/data/hooks/backup/26-conf_xmpp b/data/hooks/backup/26-conf_xmpp index 12300a00a..b55ad2bfc 100755 --- a/data/hooks/backup/26-conf_xmpp +++ b/data/hooks/backup/26-conf_xmpp @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/xmpp" diff --git a/data/hooks/backup/29-conf_nginx b/data/hooks/backup/29-conf_nginx index d900c7535..81e145e24 100755 --- a/data/hooks/backup/29-conf_nginx +++ b/data/hooks/backup/29-conf_nginx @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/nginx" diff --git a/data/hooks/backup/32-conf_cron b/data/hooks/backup/32-conf_cron index 2fea9f53f..063ec1a3f 100755 --- a/data/hooks/backup/32-conf_cron +++ b/data/hooks/backup/32-conf_cron @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/cron" diff --git a/data/hooks/backup/40-conf_ynh_currenthost b/data/hooks/backup/40-conf_ynh_currenthost index e4a684576..6a98fd0d2 100755 --- a/data/hooks/backup/40-conf_ynh_currenthost +++ b/data/hooks/backup/40-conf_ynh_currenthost @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh" diff --git a/data/hooks/conf_regen/03-ssh b/data/hooks/conf_regen/03-ssh index 9de527518..5bb9cf916 100755 --- a/data/hooks/conf_regen/03-ssh +++ b/data/hooks/conf_regen/03-ssh @@ -2,7 +2,7 @@ set -e -. /usr/share/yunohost/helpers.d/utils +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 461c10c0c..7ca63c003 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -2,7 +2,7 @@ set -e -. /usr/share/yunohost/helpers.d/utils +. /usr/share/yunohost/helpers do_init_regen() { if [[ $EUID -ne 0 ]]; then diff --git a/data/hooks/conf_regen/34-mysql b/data/hooks/conf_regen/34-mysql index 5ee91827b..9f35fec18 100755 --- a/data/hooks/conf_regen/34-mysql +++ b/data/hooks/conf_regen/34-mysql @@ -2,6 +2,7 @@ set -e MYSQL_PKG="mariadb-server-10.1" +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 @@ -15,7 +16,6 @@ do_post_regen() { regen_conf_files=$1 if [ ! -f /etc/yunohost/mysql ]; then - . /usr/share/yunohost/helpers.d/string # ensure that mysql is running sudo systemctl -q is-active mysql.service \ @@ -25,8 +25,6 @@ do_post_regen() { mysql_password=$(ynh_string_random 10) sudo mysqladmin -s -u root -pyunohost password "$mysql_password" || { if [ $FORCE -eq 1 ]; then - . /usr/share/yunohost/helpers.d/package - echo "It seems that you have already configured MySQL." \ "YunoHost needs to have a root access to MySQL to runs its" \ "applications, and is going to reset the MySQL root password." \ diff --git a/data/hooks/conf_regen/43-dnsmasq b/data/hooks/conf_regen/43-dnsmasq index 2c8ce797b..ed795c058 100755 --- a/data/hooks/conf_regen/43-dnsmasq +++ b/data/hooks/conf_regen/43-dnsmasq @@ -1,13 +1,11 @@ #!/bin/bash set -e +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 - # source ip helpers - . /usr/share/yunohost/helpers.d/ip - cd /usr/share/yunohost/templates/dnsmasq # create directory for pending conf diff --git a/data/hooks/restore/11-conf_ynh_mysql b/data/hooks/restore/11-conf_ynh_mysql index 0aaaccd54..1336a2cc2 100644 --- a/data/hooks/restore/11-conf_ynh_mysql +++ b/data/hooks/restore/11-conf_ynh_mysql @@ -1,6 +1,8 @@ backup_dir="$1/conf/ynh/mysql" MYSQL_PKG="mariadb-server-10.1" +. /usr/share/yunohost/helpers + # ensure that mysql is running service mysql status >/dev/null 2>&1 \ || service mysql start @@ -11,13 +13,11 @@ service mysql status >/dev/null 2>&1 \ new_pwd=$(sudo cat "${backup_dir}/root_pwd" || sudo cat "${backup_dir}/mysql") [ -z "$curr_pwd" ] && curr_pwd="yunohost" [ -z "$new_pwd" ] && { - . /usr/share/yunohost/helpers.d/string new_pwd=$(ynh_string_random 10) } # attempt to change it sudo mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || { - . /usr/share/yunohost/helpers.d/package echo "It seems that you have already configured MySQL." \ "YunoHost needs to have a root access to MySQL to runs its" \ From 683d62d0e5df9dd07ca569e48a4c96e9ca9856d8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 18:43:14 +0100 Subject: [PATCH 31/39] [microdecision] Fix interface with meltdown checker script, stdout contains weird debug messages when ran inside LXC :| --- src/yunohost/tools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index a220e21ca..b2fbf380c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -735,6 +735,14 @@ def _check_if_vulnerable_to_meltdown(): output, err = call.communicate() assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode + # If there are multiple lines, sounds like there was some messages + # in stdout that are not json >.> ... Try to get the actual json + # stuff which should be the last line + output = output.strip() + if "\n" in output: + logger.debug("Original meltdown checker output : %s" % output) + output = output.split("\n")[-1] + CVEs = json.loads(output) assert len(CVEs) == 1 assert CVEs[0]["NAME"] == "MELTDOWN" From d0c7603d9db1a406c203e97f95c405e6bb0d941a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 20:25:30 +0100 Subject: [PATCH 32/39] Wtf is wrong with you sudo :| Why don't you forward the damn DEBIAN_FRONTEND --- data/helpers.d/package | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/package b/data/helpers.d/package index 1b7c38c5c..e7aa3faf8 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -71,7 +71,7 @@ ynh_package_version() { # usage: ynh_apt update ynh_apt() { ynh_wait_dpkg_free - DEBIAN_FRONTEND=noninteractive sudo apt-get -y $@ + DEBIAN_FRONTEND=noninteractive apt-get -y $@ } # Update package index files @@ -150,7 +150,7 @@ ynh_package_install_from_equivs () { cp "$controlfile" "${TMPDIR}/control" (cd "$TMPDIR" equivs-build ./control 1> /dev/null - DEBIAN_FRONTEND=noninteractive dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) + dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) ynh_package_install -f || ynh_die --message="Unable to install dependencies" [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR # Remove the temp dir. From 39ae73604400ecb050af5c8df3d26d8180f236f9 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 19 Feb 2019 17:43:33 +0100 Subject: [PATCH 33/39] Ensure the tar file is closed --- src/yunohost/backup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index ed7799fc1..9268c7613 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1802,10 +1802,11 @@ class TarBackupMethod(BackupMethod): # Add the "source" into the archive and transform the path into # "dest" tar.add(path['source'], arcname=path['dest']) - tar.close() except IOError: logger.error(m18n.n('backup_archive_writing_error'), exc_info=1) raise YunohostError('backup_creation_failed') + finally: + tar.close() # Move info file shutil.copy(os.path.join(self.work_dir, 'info.json'), From a69cd443aece34d7225cb281acc2019583980985 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 19 Feb 2019 17:49:57 +0100 Subject: [PATCH 34/39] More explicit error --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 8528c2576..244eba80f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -75,7 +75,7 @@ "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'", "backup_archive_open_failed": "Unable to open the backup archive", "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup", - "backup_archive_writing_error": "Unable to add files to backup into the compressed archive", + "backup_archive_writing_error": "Unable to add files '{source:s}' (named in the archive: '{dest:s}') to backup into the compressed archive '{archive:s}'", "backup_ask_for_copying_if_needed": "Some files couldn't be prepared to be backuped using the method that avoid to temporarily waste space on the system. To perform the backup, {size:s}MB should be used temporarily. Do you agree?", "backup_borg_not_implemented": "Borg backup method is not yet implemented", "backup_cant_mount_uncompress_archive": "Unable to mount in readonly mode the uncompress archive directory", From 44455cd433eaf529e40df921fef1c33de3505086 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 19 Feb 2019 17:50:12 +0100 Subject: [PATCH 35/39] More explicit error --- src/yunohost/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 9268c7613..062343a46 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1803,7 +1803,7 @@ class TarBackupMethod(BackupMethod): # "dest" tar.add(path['source'], arcname=path['dest']) except IOError: - logger.error(m18n.n('backup_archive_writing_error'), exc_info=1) + logger.error(m18n.n('backup_archive_writing_error', source=path['source'], archive=self._archive_file, dest=path['dest']), exc_info=1) raise YunohostError('backup_creation_failed') finally: tar.close() From 3c33eb078d3f87c97b59ef7bdb1e51ddd3f2702e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 18:43:14 +0100 Subject: [PATCH 36/39] [microdecision] Fix interface with meltdown checker script, stdout contains weird debug messages when ran inside LXC :| --- src/yunohost/tools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 915b63940..4d4f5fffe 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -735,6 +735,14 @@ def _check_if_vulnerable_to_meltdown(): output, err = call.communicate() assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode + # If there are multiple lines, sounds like there was some messages + # in stdout that are not json >.> ... Try to get the actual json + # stuff which should be the last line + output = output.strip() + if "\n" in output: + logger.debug("Original meltdown checker output : %s" % output) + output = output.split("\n")[-1] + CVEs = json.loads(output) assert len(CVEs) == 1 assert CVEs[0]["NAME"] == "MELTDOWN" From ce79dd8c8a3ca1f16dda46cfd749714e3eb1fc07 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 19 Feb 2019 19:12:59 +0100 Subject: [PATCH 37/39] Update changelog for 3.4.2.4 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3f3a6a5ef..7be4212fe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.4.2.4) stable; urgency=low + + - [fix] Meltdown vulnerability checker something outputing trash instead of pure json + + -- Alexandre Aubin Tue, 19 Feb 2019 19:11:38 +0000 + yunohost (3.4.2.3) stable; urgency=low - [fix] Admin password appearing in logs after logging in on webadmin From 5c5330be3da369e1a255bb305009d35738927a8e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 21 Feb 2019 15:35:49 +0100 Subject: [PATCH 38/39] Issue happening with some weird app instance name ... --- src/yunohost/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0bca68787..302049ed8 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2345,6 +2345,7 @@ def _parse_app_instance_name(app_instance_name): True """ match = re_app_instance_name.match(app_instance_name) + assert match, "Could not parse app instance name : %s" % app_instance_name appid = match.groupdict().get('appid') app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1 return (appid, app_instance_nb) From bca2af3391a815873a7b63084d941640e0dbadf0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 21 Feb 2019 23:37:19 +0100 Subject: [PATCH 39/39] [microdecision] I'm sick of those people who end up with app repo being added as app list and messing up everything -.- --- src/yunohost/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 302049ed8..5f5d9f8f9 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -97,6 +97,9 @@ def app_fetchlist(url=None, name=None): name -- Name of the list url -- URL of remote JSON list """ + if not url.endswith(".json"): + raise YunohostError("This is not a valid application list url. It should end with .json.") + # If needed, create folder where actual appslists are stored if not os.path.exists(REPO_PATH): os.makedirs(REPO_PATH)