From 92bf413d684d8c3aea0e00833a515a82f7070e2e Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 31 Mar 2017 10:02:40 +0200 Subject: [PATCH 001/113] [fix] Remove version from api --- bin/yunohost-api | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/yunohost-api b/bin/yunohost-api index d2b219f8b..054d5df84 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -192,12 +192,10 @@ if __name__ == '__main__': _init_moulinette(opts.use_websocket, opts.debug, opts.verbose) # Run the server - from yunohost.utils.packages import ynh_packages_version ret = moulinette.api( _retrieve_namespaces(), host=opts.host, port=opts.port, routes={ ('GET', '/installed'): is_installed, - ('GET', '/version'): ynh_packages_version, }, use_cache=opts.use_cache, use_websocket=opts.use_websocket ) sys.exit(ret) From 35d4a1001fcd04207a9341389b5614e55919229d Mon Sep 17 00:00:00 2001 From: Jimmy Monin Date: Sat, 2 Sep 2017 18:27:25 +0200 Subject: [PATCH 002/113] Add fail2ban helpers --- data/helpers.d/backend | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index c54e82754..0795be38e 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -183,3 +183,57 @@ ynh_remove_fpm_config () { ynh_secure_remove "/etc/php5/fpm/conf.d/20-$app.ini" 2>&1 sudo systemctl reload php5-fpm } + +# Create a dedicated fail2ban config (jail and filter conf files) +# +# usage: ynh_add_fail2ban_config log_file filter [max_retry [ports]] +# | arg: log_file - Log file to be checked by fail2ban +# | arg: failregex - Failregex to be looked for by fail2ban +# | arg: max_retry - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: ports - Ports blocked for a banned IP address - default: http,https +ynh_add_fail2ban_config () { + # Process parameters + logpath=$1 + failregex=$2 + max_retry=${3:-3} + ports=${4:-http,https} + + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" + finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" + ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 + ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 + + sudo tee $finalfail2banjailconf < Date: Sat, 19 May 2018 10:11:16 +0200 Subject: [PATCH 003/113] Update to latest version from Experimental_helpers --- data/helpers.d/backend | 44 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 0795be38e..e14095102 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -192,41 +192,47 @@ ynh_remove_fpm_config () { # | arg: max_retry - Maximum number of retries allowed before banning IP address - default: 3 # | arg: ports - Ports blocked for a banned IP address - default: http,https ynh_add_fail2ban_config () { - # Process parameters - logpath=$1 - failregex=$2 - max_retry=${3:-3} - ports=${4:-http,https} - - test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." - + # Process parameters + logpath=$1 + failregex=$2 + max_retry=${3:-3} + ports=${4:-http,https} + + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 - - sudo tee $finalfail2banjailconf <&2 + echo "WARNING${fail2ban_error#*WARNING}" >&2 + fi } # Remove the dedicated fail2ban config (jail and filter conf files) @@ -234,6 +240,6 @@ EOF # usage: ynh_remove_fail2ban_config ynh_remove_fail2ban_config () { ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" - ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" - sudo systemctl restart fail2ban -} \ No newline at end of file + ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" + systemctl reload fail2ban +} From 7752bc0fb7eab0811926c891be5bb9703cd3b23a Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 27 Aug 2018 21:51:36 +0200 Subject: [PATCH 004/113] Update fail2ban helpers from experimental helpers. --- data/helpers.d/backend | 73 +++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index e14095102..4e939331e 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -187,26 +187,31 @@ ynh_remove_fpm_config () { # Create a dedicated fail2ban config (jail and filter conf files) # # usage: ynh_add_fail2ban_config log_file filter [max_retry [ports]] -# | arg: log_file - Log file to be checked by fail2ban -# | arg: failregex - Failregex to be looked for by fail2ban -# | arg: max_retry - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: ports - Ports blocked for a banned IP address - default: http,https +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https ynh_add_fail2ban_config () { - # Process parameters - logpath=$1 - failregex=$2 - max_retry=${3:-3} - ports=${4:-http,https} - - test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + # Declare an array to define the options of this helper. + declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= ) + local logpath + local failregex + local max_retry + local ports + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + max_retry=${max_retry:-3} + ports=${ports:-http,https} - finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" - finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" - ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 - ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." - tee $finalfail2banjailconf <&2 - echo "WARNING${fail2ban_error#*WARNING}" >&2 - fi + if [ "$(lsb_release --codename --short)" != "jessie" ]; then + systemctl reload fail2ban + else + systemctl restart fail2ban + fi + local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")" + if [ -n "$fail2ban_error" ] + then + echo "[ERR] Fail2ban failed to load the jail for $app" >&2 + echo "WARNING${fail2ban_error#*WARNING}" >&2 + fi } # Remove the dedicated fail2ban config (jail and filter conf files) # # usage: ynh_remove_fail2ban_config ynh_remove_fail2ban_config () { - ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" - ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" - systemctl reload fail2ban + ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" + ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" + if [ "$(lsb_release --codename --short)" != "jessie" ]; then + systemctl reload fail2ban + else + systemctl restart fail2ban + fi } From 0fd9e179f16a92af63292e9e8b048623fda5ac2e Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 28 Aug 2018 00:14:37 +0200 Subject: [PATCH 005/113] Add ynh_check_app_version_changed - Add the new helper ynh_check_app_version_changed to check the version before an upgrade - Add also the helpers ynh_read_manifest, ynh_app_upstream_version and ynh_app_package_version. - These previous helper have been modified (from the experimental version) to support getopts - ynh_check_app_version_changed has been modified to use ynh_app_upstream_version --- data/helpers.d/system | 105 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/data/helpers.d/system b/data/helpers.d/system index 70cc57493..63b90ef13 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -53,3 +53,108 @@ ynh_abort_if_errors () { ynh_get_debian_release () { echo $(lsb_release --codename --short) } + +# Read the value of a key in a ynh manifest file +# +# usage: ynh_read_manifest manifest key +# | arg: -m, --manifest= - Path of the manifest to read +# | arg: -k, --key= - Name of the key to find +ynh_read_manifest () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=manifest= [k]=manifest_key= ) + local manifest + local manifest_key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + python3 -c "import sys, json;print(json.load(open('$manifest', encoding='utf-8'))['$manifest_key'])" +} + +# Read the upstream version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number before ~ynh +# In the last example it return 4.3-2 +# +# usage: ynh_app_upstream_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +ynh_app_upstream_version () { + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + if [ ! -e "$manifest_path" ]; then + manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + fi + version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + echo "${version_key/~ynh*/}" +} + +# Read package version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number after ~ynh +# In the last example it return 3 +# +# usage: ynh_app_package_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +ynh_app_package_version () { + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + if [ ! -e "$manifest_path" ]; then + manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + fi + version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + echo "${version_key/*~ynh/}" +} + +# Checks the app version to upgrade with the existing app version and returns: +# - UPGRADE_APP if the upstream app version has changed +# - UPGRADE_PACKAGE if only the YunoHost package has changed +# +## It stops the current script without error if the package is up-to-date +# +# This helper should be used to avoid an upgrade of an app, or the upstream part +# of it, when it's not needed +# +# To force an upgrade, even if the package is up to date, +# you have to set the variable YNH_FORCE_UPGRADE before. +# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp + +# usage: ynh_check_app_version_changed +ynh_check_app_version_changed () { + local force_upgrade=${YNH_FORCE_UPGRADE:-0} + local package_check=${PACKAGE_CHECK_EXEC:-0} + + # By default, upstream app version has changed + local return_value="UPGRADE_APP" + + local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0) + local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")" + local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0) + local update_upstream_version="$(ynh_app_upstream_version)" + + if [ "$current_version" == "$update_version" ] ; then + # Complete versions are the same + if [ "$force_upgrade" != "0" ] + then + echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2 + unset YNH_FORCE_UPGRADE + elif [ "$package_check" != "0" ] + then + echo "Upgrade forced for package check." >&2 + else + ynh_die "Up-to-date, nothing to do" 0 + fi + elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then + # Upstream versions are the same, only YunoHost package versions differ + return_value="UPGRADE_PACKAGE" + fi + echo $return_value +} From e0533a1a6a48bbce12fe8a961647f62436fa3649 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Aug 2018 00:29:24 +0200 Subject: [PATCH 006/113] Really important contribution to the PR :ninja: --- data/helpers.d/system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 63b90ef13..f09343953 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -126,7 +126,7 @@ ynh_app_package_version () { # To force an upgrade, even if the package is up to date, # you have to set the variable YNH_FORCE_UPGRADE before. # example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp - +# # usage: ynh_check_app_version_changed ynh_check_app_version_changed () { local force_upgrade=${YNH_FORCE_UPGRADE:-0} From 62c8f577c9f57e9164a7bbbaffd5a5a611113daa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 13:25:29 +0200 Subject: [PATCH 007/113] Remove old sudo --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 4e939331e..bc3ae6a7d 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -211,7 +211,7 @@ ynh_add_fail2ban_config () { ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 - sudo tee $finalfail2banjailconf < Date: Wed, 29 Aug 2018 13:27:25 +0200 Subject: [PATCH 008/113] Remove sudo --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index bc3ae6a7d..6b5ca1d37 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -220,7 +220,7 @@ logpath = $logpath maxretry = $max_retry EOF - sudo tee $finalfail2banfilterconf < Date: Wed, 10 Oct 2018 21:15:06 +0200 Subject: [PATCH 009/113] Use getopts for ynh_use_logrotate --- data/helpers.d/backend | 50 ++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 94e26350c..d67d36333 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -1,9 +1,11 @@ +#!/bin/bash + # Use logrotate to manage the logfile # -# usage: ynh_use_logrotate [logfile] [--non-append|--append] [specific_user/specific_group] -# | arg: logfile - absolute path of logfile -# | arg: --non-append - (Option) Replace the config file instead of appending this new config. -# | arg: specific_user : run logrotate as the specified user and group. If not specified logrotate is runned as root. +# usage: ynh_use_logrotate [--logfile=/log/file] [--nonappend] [--specific_user=user/group] +# | arg: -l, --logfile= - absolute path of logfile +# | arg: -n, --nonappend - (Option) Replace the config file instead of appending this new config. +# | arg: -u, --specific_user : run logrotate as the specified user and group. If not specified logrotate is runned as root. # # If no argument provided, a standard directory will be use. /var/log/${app} # You can provide a path with the directory only or with the logfile. @@ -13,28 +15,52 @@ # It's possible to use this helper several times, each config will be added to the same logrotate config file. # Unless you use the option --non-append ynh_use_logrotate () { - local customtee="tee -a" - local user_group="${3:-}" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [l]=logfile= [n]=nonappend [u]=specific_user= [y]=non [a]=append ) + # [y]=non [a]=append are only for legacy purpose, to not fail on the old option '--non-append' + local logfile + local nonappend + local specific_user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local logfile="${logfile:-}" + local nonappend="${nonappend:-0}" + local specific_user="${specific_user:-}" + + # LEGACY CODE - PRE GETOPTS if [ $# -gt 0 ] && [ "$1" == "--non-append" ]; then - customtee="tee" + nonappend=1 # Destroy this argument for the next command. shift elif [ $# -gt 1 ] && [ "$2" == "--non-append" ]; then - customtee="tee" + nonappend=1 fi - if [ $# -gt 0 ]; then + + if [ $# -gt 0 ] && [ "$(echo ${1:0:1})" != "-" ]; then if [ "$(echo ${1##*.})" == "log" ]; then # Keep only the extension to check if it's a logfile local logfile=$1 # In this case, focus logrotate on the logfile else local logfile=$1/*.log # Else, uses the directory and all logfile into it. fi + fi + # LEGACY CODE + + local customtee="tee -a" + if [ "$nonappend" -eq 1 ]; then + customtee="tee" + fi + if [ -n "$logfile" ] + then + if [ "$(echo ${logfile##*.})" != "log" ]; then # Keep only the extension to check if it's a logfile + local logfile="$1/*.log" # Else, uses the directory and all logfile into it. + fi else - local logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log + logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log fi local su_directive="" - if [[ -n $user_group ]]; then + if [[ -n $specific_user ]]; then su_directive=" # Run logorotate as specific user - group - su ${user_group%/*} ${user_group#*/}" + su ${specific_user%/*} ${specific_user#*/}" fi cat > ./${app}-logrotate << EOF # Build a config file for logrotate From f354c4b8332fc6d0d6ed4d293d37c8175570292c Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 16 Oct 2018 16:57:24 +0200 Subject: [PATCH 010/113] Use getopts for ynh_add_systemd_config and ynh_remove_systemd_config --- data/helpers.d/backend | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index d67d36333..5103998a2 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -99,9 +99,9 @@ ynh_remove_logrotate () { # Create a dedicated systemd config # -# usage: ynh_add_systemd_config [service] [template] -# | arg: service - Service name (optionnal, $app by default) -# | arg: template - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template) +# usage: ynh_add_systemd_config [--service=service] [--template=template] +# | arg: -s, --service - Service name (optionnal, $app by default) +# | arg: -t, --template - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template) # # This will use the template ../conf/.service # to generate a systemd config, by replacing the following keywords @@ -112,11 +112,18 @@ ynh_remove_logrotate () { # __FINALPATH__ by $final_path # ynh_add_systemd_config () { - local service_name="${1:-$app}" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [s]=service= [t]=template ) + local service + local template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local service="${service:-$app}" + local template="${nonappend:-systemd.service}" - finalsystemdconf="/etc/systemd/system/$service_name.service" + finalsystemdconf="/etc/systemd/system/$service.service" ynh_backup_if_checksum_is_different "$finalsystemdconf" - sudo cp ../conf/${2:-systemd.service} "$finalsystemdconf" + sudo cp ../conf/$template "$finalsystemdconf" # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. # Substitute in a nginx config file only if the variable is not empty @@ -129,22 +136,27 @@ ynh_add_systemd_config () { ynh_store_file_checksum "$finalsystemdconf" sudo chown root: "$finalsystemdconf" - sudo systemctl enable $service_name + sudo systemctl enable $service sudo systemctl daemon-reload } # Remove the dedicated systemd config # -# usage: ynh_remove_systemd_config [service] -# | arg: service - Service name (optionnal, $app by default) +# usage: ynh_remove_systemd_config [--service=service] +# | arg: -s, --service - Service name (optionnal, $app by default) # ynh_remove_systemd_config () { - local service_name="${1:-$app}" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [s]=service= ) + local service + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local service="${service:-$app}" - local finalsystemdconf="/etc/systemd/system/$service_name.service" + local finalsystemdconf="/etc/systemd/system/$service.service" if [ -e "$finalsystemdconf" ]; then - sudo systemctl stop $service_name - sudo systemctl disable $service_name + sudo systemctl stop $service + sudo systemctl disable $service ynh_secure_remove "$finalsystemdconf" sudo systemctl daemon-reload fi From d59401f08e7ee5a0007f6060a077d37464de0c5d Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 16 Oct 2018 23:41:11 +0200 Subject: [PATCH 011/113] Use getopts for helpers in filesystem --- data/helpers.d/filesystem | 176 ++++++++++++++++++++++---------------- 1 file changed, 103 insertions(+), 73 deletions(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index c07de2ece..c9c58e2fe 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -10,13 +10,12 @@ CAN_BIND=${CAN_BIND:-1} # # If DEST is ended by a slash it complete this path with the basename of SRC. # -# usage: ynh_backup src [dest [is_big [arg]]] -# | arg: src - file or directory to bind or symlink or copy. it shouldn't be in +# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] +# | arg: -s, --src_path - file or directory to bind or symlink or copy. it shouldn't be in # the backup dir. -# | arg: dest - destination file or directory inside the +# | arg: -d, --dest_path - destination file or directory inside the # backup dir -# | arg: is_big - 1 to indicate data are big (mail, video, image ...) -# | arg: arg - Deprecated arg +# | arg: -b, --is_big - Indicate data are big (mail, video, image ...) # # example: # # Wordpress app context @@ -43,15 +42,23 @@ CAN_BIND=${CAN_BIND:-1} # ynh_backup() { # TODO find a way to avoid injection by file strange naming ! - local SRC_PATH="$1" - local DEST_PATH="${2:-}" - local IS_BIG="${3:-0}" + + # Declare an array to define the options of this helper. + declare -Ar args_array=( [s]=src_path= [d]=dest_path [b]=is_big ) + local src_path + local dest_path + local is_big + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local dest_path="${dest_path:-}" + local is_big="${is_big:-0}" + BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items - if [ "$IS_BIG" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then - echo "$SRC_PATH will not be saved, because backup_core_only is set." >&2 + if [ "$is_big" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then + echo "$src_path will not be saved, because backup_core_only is set." >&2 return 0 fi @@ -59,24 +66,24 @@ ynh_backup() { # Format correctly source and destination paths # ============================================================================== # Be sure the source path is not empty - [[ -e "${SRC_PATH}" ]] || { - echo "Source path '${SRC_PATH}' does not exist" >&2 + [[ -e "${src_path}" ]] || { + echo "Source path '${src_path}' does not exist" >&2 return 1 } # Transform the source path as an absolute path # If it's a dir remove the ending / - SRC_PATH=$(realpath "$SRC_PATH") + src_path=$(realpath "$src_path") # If there is no destination path, initialize it with the source path # relative to "/". - # eg: SRC_PATH=/etc/yunohost -> DEST_PATH=etc/yunohost - if [[ -z "$DEST_PATH" ]]; then + # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost + if [[ -z "$dest_path" ]]; then - DEST_PATH="${SRC_PATH#/}" + dest_path="${src_path#/}" else - if [[ "${DEST_PATH:0:1}" == "/" ]]; then + if [[ "${dest_path:0:1}" == "/" ]]; then # If the destination path is an absolute path, transform it as a path # relative to the current working directory ($YNH_CWD) @@ -85,43 +92,43 @@ ynh_backup() { # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/ # # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR - DEST_PATH="${DEST_PATH#$YNH_CWD/}" + dest_path="${dest_path#$YNH_CWD/}" # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD - [[ "${DEST_PATH:0:1}" == "/" ]] \ - && DEST_PATH="${DEST_PATH#/}" + [[ "${dest_path:0:1}" == "/" ]] \ + && dest_path="${dest_path#/}" fi - # Complete DEST_PATH if ended by a / - [[ "${DEST_PATH: -1}" == "/" ]] \ - && DEST_PATH="${DEST_PATH}/$(basename $SRC_PATH)" + # Complete dest_path if ended by a / + [[ "${dest_path: -1}" == "/" ]] \ + && dest_path="${dest_path}/$(basename $src_path)" fi - # Check if DEST_PATH already exists in tmp archive - [[ ! -e "${DEST_PATH}" ]] || { - echo "Destination path '${DEST_PATH}' already exist" >&2 + # Check if dest_path already exists in tmp archive + [[ ! -e "${dest_path}" ]] || { + echo "Destination path '${dest_path}' already exist" >&2 return 1 } # Add the relative current working directory to the destination path - local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR}" - REL_DIR="${REL_DIR%/}/" - DEST_PATH="${REL_DIR}${DEST_PATH}" - DEST_PATH="${DEST_PATH#/}" + local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}" + rel_dir="${rel_dir%/}/" + dest_path="${rel_dir}${dest_path}" + dest_path="${dest_path#/}" # ============================================================================== # ============================================================================== # Write file to backup into backup_list # ============================================================================== - local SRC=$(echo "${SRC_PATH}" | sed -r 's/"/\"\"/g') - local DEST=$(echo "${DEST_PATH}" | sed -r 's/"/\"\"/g') - echo "\"${SRC}\",\"${DEST}\"" >> "${YNH_BACKUP_CSV}" + local src=$(echo "${src_path}" | sed -r 's/"/\"\"/g') + local dest=$(echo "${dest_path}" | sed -r 's/"/\"\"/g') + echo "\"${src}\",\"${dest}\"" >> "${YNH_BACKUP_CSV}" # ============================================================================== # Create the parent dir of the destination path # It's for retro compatibility, some script consider ynh_backup creates this dir - mkdir -p $(dirname "$YNH_BACKUP_DIR/${DEST_PATH}") + mkdir -p $(dirname "$YNH_BACKUP_DIR/${dest_path}") } # Restore all files linked to the restore hook or to the restore app script @@ -168,10 +175,10 @@ with open(sys.argv[1], 'r') as backup_file: # Use the registered path in backup_list by ynh_backup to restore the file at # the good place. # -# usage: ynh_restore_file ORIGIN_PATH [ DEST_PATH ] -# | arg: ORIGIN_PATH - Path where was located the file or the directory before +# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] +# | arg: -o, --origin_path - Path where was located the file or the directory before # to be backuped or relative path to $YNH_CWD where it is located in the backup archive -# | arg: DEST_PATH - Path where restore the file or the dir, if unspecified, +# | arg: -d, --dest_path - Path where restore the file or the dir, if unspecified, # the destination will be ORIGIN_PATH or if the ORIGIN_PATH doesn't exist in # the archive, the destination will be searched into backup.csv # @@ -189,43 +196,50 @@ with open(sys.argv[1], 'r') as backup_file: # ynh_restore_file "conf/nginx.conf" # ynh_restore_file () { - local ORIGIN_PATH="/${1#/}" - local ARCHIVE_PATH="$YNH_CWD${ORIGIN_PATH}" - # Default value for DEST_PATH = /$ORIGIN_PATH - local DEST_PATH="${2:-$ORIGIN_PATH}" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [o]=origin_path= [d]=dest_path= ) + local origin_path + local archive_path + local dest_path + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local origin_path="/${origin_path#/}" + local archive_path="$YNH_CWD${origin_path}" + # Default value for dest_path = /$origin_path + local dest_path="${dest_path:-$origin_path}" - # If ARCHIVE_PATH doesn't exist, search for a corresponding path in CSV - if [ ! -d "$ARCHIVE_PATH" ] && [ ! -f "$ARCHIVE_PATH" ] && [ ! -L "$ARCHIVE_PATH" ]; then - ARCHIVE_PATH="$YNH_BACKUP_DIR/$(_get_archive_path \"$ORIGIN_PATH\")" + # If archive_path doesn't exist, search for a corresponding path in CSV + if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then + archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")" fi # Move the old directory if it already exists - if [[ -e "${DEST_PATH}" ]] + if [[ -e "${dest_path}" ]] then # Check if the file/dir size is less than 500 Mo - if [[ $(du -sb ${DEST_PATH} | cut -d"/" -f1) -le "500000000" ]] + if [[ $(du -sb ${dest_path} | cut -d"/" -f1) -le "500000000" ]] then - local backup_file="/home/yunohost.conf/backup/${DEST_PATH}.backup.$(date '+%Y%m%d.%H%M%S')" + local backup_file="/home/yunohost.conf/backup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')" mkdir -p "$(dirname "$backup_file")" - mv "${DEST_PATH}" "$backup_file" # Move the current file or directory + mv "${dest_path}" "$backup_file" # Move the current file or directory else - ynh_secure_remove ${DEST_PATH} + ynh_secure_remove ${dest_path} fi fi - # Restore ORIGIN_PATH into DEST_PATH - mkdir -p $(dirname "$DEST_PATH") + # Restore origin_path into dest_path + mkdir -p $(dirname "$dest_path") # Do a copy if it's just a mounting point if mountpoint -q $YNH_BACKUP_DIR; then - if [[ -d "${ARCHIVE_PATH}" ]]; then - ARCHIVE_PATH="${ARCHIVE_PATH}/." - mkdir -p "$DEST_PATH" + if [[ -d "${archive_path}" ]]; then + archive_path="${archive_path}/." + mkdir -p "$dest_path" fi - cp -a "$ARCHIVE_PATH" "${DEST_PATH}" + cp -a "$archive_path" "${dest_path}" # Do a move if YNH_BACKUP_DIR is already a copy else - mv "$ARCHIVE_PATH" "${DEST_PATH}" + mv "$archive_path" "${dest_path}" fi } @@ -265,11 +279,17 @@ properly with chmod/chown." >&2 # # $app should be defined when calling this helper # -# usage: ynh_store_file_checksum file -# | arg: file - The file on which the checksum will performed, then stored. +# usage: ynh_store_file_checksum --file=file +# | arg: -f, --file - The file on which the checksum will performed, then stored. ynh_store_file_checksum () { - local checksum_setting_name=checksum_${1//[\/ ]/_} # Replace all '/' and ' ' by '_' - ynh_app_setting_set $app $checksum_setting_name $(sudo md5sum "$1" | cut -d' ' -f1) + # Declare an array to define the options of this helper. + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + ynh_app_setting_set $app $checksum_setting_name $(sudo md5sum "$file" | cut -d' ' -f1) } # Verify the checksum and backup the file if it's different @@ -278,12 +298,17 @@ ynh_store_file_checksum () { # # $app should be defined when calling this helper # -# usage: ynh_backup_if_checksum_is_different file -# | arg: file - The file on which the checksum test will be perfomed. +# usage: ynh_backup_if_checksum_is_different --file=file +# | arg: -f, --file - The file on which the checksum test will be perfomed. # # | ret: Return the name a the backup file, or nothing ynh_backup_if_checksum_is_different () { - local file=$1 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get $app $checksum_setting_name) if [ -n "$checksum_value" ] @@ -318,28 +343,33 @@ ynh_delete_file_checksum () { # Remove a file or a directory securely # -# usage: ynh_secure_remove path_to_remove -# | arg: path_to_remove - File or directory to remove +# usage: ynh_secure_remove file=path_to_remove +# | arg: -f, --file - File or directory to remove ynh_secure_remove () { - local path_to_remove=$1 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local forbidden_path=" \ /var/www \ /home/yunohost.app" - if [[ "$forbidden_path" =~ "$path_to_remove" \ + if [[ "$forbidden_path" =~ "$file" \ # Match all paths or subpaths in $forbidden_path - || "$path_to_remove" =~ ^/[[:alnum:]]+$ \ + || "$file" =~ ^/[[:alnum:]]+$ \ # Match all first level paths from / (Like /var, /root, etc...) - || "${path_to_remove:${#path_to_remove}-1}" = "/" ]] + || "${file:${#file}-1}" = "/" ]] # Match if the path finishes by /. Because it seems there is an empty variable then - echo "Avoid deleting $path_to_remove." >&2 + echo "Avoid deleting $file." >&2 else - if [ -e "$path_to_remove" ] + if [ -e "$file" ] then - sudo rm -R "$path_to_remove" + sudo rm -R "$file" else - echo "$path_to_remove wasn't deleted because it doesn't exist." >&2 + echo "$file wasn't deleted because it doesn't exist." >&2 fi fi } From 4c82dad2bb48a550687c48618b52b8a7e70323f8 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 16 Oct 2018 23:46:54 +0200 Subject: [PATCH 012/113] Use getopts for ip's helpers --- data/helpers.d/ip | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/data/helpers.d/ip b/data/helpers.d/ip index 092cdff4b..09c974782 100644 --- a/data/helpers.d/ip +++ b/data/helpers.d/ip @@ -1,6 +1,6 @@ # Validate an IP address # -# usage: ynh_validate_ip [family] [ip_address] +# usage: ynh_validate_ip --family=family --ip_address=ip_address # | ret: 0 for valid ip addresses, 1 otherwise # # example: ynh_validate_ip 4 111.222.333.444 @@ -9,17 +9,21 @@ ynh_validate_ip() { # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 - local IP_ADDRESS_FAMILY=$1 - local IP_ADDRESS=$2 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [f]=family= [i]=ip_address= ) + local family + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" - [ "$IP_ADDRESS_FAMILY" == "4" ] || [ "$IP_ADDRESS_FAMILY" == "6" ] || return 1 + [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 python /dev/stdin << EOF import socket import sys family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 } try: - socket.inet_pton(family["$IP_ADDRESS_FAMILY"], "$IP_ADDRESS") + socket.inet_pton(family["$family"], "$ip_address") except socket.error: sys.exit(1) sys.exit(0) @@ -30,12 +34,18 @@ EOF # # example: ynh_validate_ip4 111.222.333.444 # -# usage: ynh_validate_ip4 +# usage: ynh_validate_ip4 --ip_address=ip_address # | ret: 0 for valid ipv4 addresses, 1 otherwise # ynh_validate_ip4() { - ynh_validate_ip 4 $1 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [i]=ip_address= ) + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_validate_ip 4 $ip_address } @@ -43,10 +53,16 @@ ynh_validate_ip4() # # example: ynh_validate_ip6 2000:dead:beef::1 # -# usage: ynh_validate_ip6 +# usage: ynh_validate_ip6 --ip_address=ip_address # | ret: 0 for valid ipv6 addresses, 1 otherwise # ynh_validate_ip6() { - ynh_validate_ip 6 $1 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [i]=ip_address= ) + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_validate_ip 6 $ip_address } From 35b483a7c083a1aa86865b21a0bbfeedef6c9b28 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 17 Oct 2018 00:05:42 +0200 Subject: [PATCH 013/113] Use getopts for sql's helpers --- data/helpers.d/mysql | 130 ++++++++++++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 38 deletions(-) diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 7bc93fad5..0cfe65380 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -5,32 +5,57 @@ MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql # example: ynh_mysql_connect_as 'user' 'pass' <<< "UPDATE ...;" # example: ynh_mysql_connect_as 'user' 'pass' < /path/to/file.sql # -# usage: ynh_mysql_connect_as user pwd [db] -# | arg: user - the user name to connect as -# | arg: pwd - the user password -# | arg: db - the database to connect to +# usage: ynh_mysql_connect_as --user=user --password=password [--database=database] +# | arg: -u, --user - the user name to connect as +# | arg: -p, --password - the user password +# | arg: -d, --database - the database to connect to ynh_mysql_connect_as() { - mysql -u "$1" --password="$2" -B "${3:-}" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=user= [p]=password= [d]=database= ) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + mysql -u "$user" --password="$password" -B "$database" } # Execute a command as root user # -# usage: ynh_mysql_execute_as_root sql [db] -# | arg: sql - the SQL command to execute -# | arg: db - the database to connect to +# usage: ynh_mysql_execute_as_root --sql=sql [--database=database] +# | arg: -s, --sql - the SQL command to execute +# | arg: -d, --database - the database to connect to ynh_mysql_execute_as_root() { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [s]=sql= [d]=database= ) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + ynh_mysql_connect_as "root" "$(sudo cat $MYSQL_ROOT_PWD_FILE)" \ - "${2:-}" <<< "$1" + "$database" <<< "$sql" } # Execute a command from a file as root user # -# usage: ynh_mysql_execute_file_as_root file [db] -# | arg: file - the file containing SQL commands -# | arg: db - the database to connect to +# usage: ynh_mysql_execute_file_as_root --file=file [--database=database] +# | arg: -f, --file - the file containing SQL commands +# | arg: -d, --database - the database to connect to ynh_mysql_execute_file_as_root() { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [f]=file= [d]=database= ) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + ynh_mysql_connect_as "root" "$(sudo cat $MYSQL_ROOT_PWD_FILE)" \ - "${2:-}" < "$1" + "$database" < "$file" } # Create a database and grant optionnaly privilegies to a user @@ -73,11 +98,17 @@ ynh_mysql_drop_db() { # # example: ynh_mysql_dump_db 'roundcube' > ./dump.sql # -# usage: ynh_mysql_dump_db db -# | arg: db - the database name to dump +# usage: ynh_mysql_dump_db --database=database +# | arg: -d, --database - the database name to dump # | ret: the mysqldump output ynh_mysql_dump_db() { - mysqldump -u "root" -p"$(sudo cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$1" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [d]=database= ) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + mysqldump -u "root" -p"$(sudo cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$database" } # Create a user @@ -94,11 +125,16 @@ ynh_mysql_create_user() { # Check if a mysql user exists # -# usage: ynh_mysql_user_exists user -# | arg: user - the user for which to check existence +# usage: ynh_mysql_user_exists --user=user +# | arg: -u, --user - the user for which to check existence ynh_mysql_user_exists() { - local user=$1 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=user= ) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + if [[ -z $(ynh_mysql_execute_as_root "SELECT User from mysql.user WHERE User = '$user';") ]] then return 1 @@ -122,28 +158,40 @@ ynh_mysql_drop_user() { # After executing this helper, the password of the created database will be available in $db_pwd # It will also be stored as "mysqlpwd" into the app settings. # -# usage: ynh_mysql_setup_db user name [pwd] -# | arg: user - Owner of the database -# | arg: name - Name of the database -# | arg: pwd - Password of the database. If not given, a password will be generated +# usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_password=pwd] +# | arg: -u, --db_user - Owner of the database +# | arg: -n, --db_name - Name of the database +# | arg: -p, --db_password - Password of the database. If not given, a password will be generated ynh_mysql_setup_db () { - local db_user="$1" - local db_name="$2" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=db_user= [n]=db_name= [p]=db_password= ) + local db_user + local db_name + local db_password + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local new_db_pwd=$(ynh_string_random) # Generate a random password - # If $3 is not given, use new_db_pwd instead for db_pwd. - db_pwd="${3:-$new_db_pwd}" - ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database - ynh_app_setting_set $app mysqlpwd $db_pwd # Store the password in the app's config + # If $db_password is not given, use new_db_pwd instead for db_password. + db_password="${db_password:-$new_db_pwd}" + + ynh_mysql_create_db "$db_name" "$db_user" "$db_password" # Create the database + ynh_app_setting_set $app mysqlpwd $db_password # Store the password in the app's config } # Remove a database if it exists, and the associated user # -# usage: ynh_mysql_remove_db user name -# | arg: user - Owner of the database -# | arg: name - Name of the database +# usage: ynh_mysql_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user - Owner of the database +# | arg: -n, --db_name - Name of the database ynh_mysql_remove_db () { - local db_user="$1" - local db_name="$2" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=db_user= [n]=db_name= ) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local mysql_root_password=$(sudo cat $MYSQL_ROOT_PWD_FILE) if mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name"; then # Check if the database exists echo "Removing database $db_name" >&2 @@ -163,10 +211,16 @@ ynh_mysql_remove_db () { # # example: dbname=$(ynh_sanitize_dbid $app) # -# usage: ynh_sanitize_dbid name -# | arg: name - name to correct/sanitize +# usage: ynh_sanitize_dbid --db_name=name +# | arg: -n, --db_name - name to correct/sanitize # | ret: the corrected name ynh_sanitize_dbid () { - local dbid=${1//[-.]/_} # We should avoid having - and . in the name of databases. They are replaced by _ - echo $dbid + # Declare an array to define the options of this helper. + declare -Ar args_array=( [n]=db_name= ) + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # We should avoid having - and . in the name of databases. They are replaced by _ + echo ${db_name//[-.]/_} } From 743bc0d436b893375f08e62102c5ddc5be0fa069 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 17 Oct 2018 00:11:03 +0200 Subject: [PATCH 014/113] Fix bad copy paste --- data/helpers.d/filesystem | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index c9c58e2fe..34b0487cc 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -44,7 +44,7 @@ ynh_backup() { # TODO find a way to avoid injection by file strange naming ! # Declare an array to define the options of this helper. - declare -Ar args_array=( [s]=src_path= [d]=dest_path [b]=is_big ) + declare -Ar args_array=( [s]=src_path= [d]=dest_path= [b]=is_big ) local src_path local dest_path local is_big From 2840531cf44b25c042af7e3dc6c007cae31105b0 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 17 Oct 2018 00:11:08 +0200 Subject: [PATCH 015/113] Fix bad copy paste --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 5103998a2..025ce45f0 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -113,7 +113,7 @@ ynh_remove_logrotate () { # ynh_add_systemd_config () { # Declare an array to define the options of this helper. - declare -Ar args_array=( [s]=service= [t]=template ) + declare -Ar args_array=( [s]=service= [t]=template= ) local service local template # Manage arguments with getopts From a701b425199fd6547919b0882288cde36f2d71ba Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 21 Dec 2018 20:40:02 +0100 Subject: [PATCH 016/113] Use getopts for network's helpers --- data/helpers.d/network | 66 +++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/data/helpers.d/network b/data/helpers.d/network index f9e37e6cc..fc7eb3f69 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -8,10 +8,15 @@ # ynh_normalize_url_path /example/ -> /example # ynh_normalize_url_path / -> / # -# usage: ynh_normalize_url_path path_to_normalize -# | arg: url_path_to_normalize - URL path to normalize before using it +# usage: ynh_normalize_url_path --path_url=path_to_normalize +# | arg: -p, --path_url - URL path to normalize before using it ynh_normalize_url_path () { - local path_url=$1 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [p]=path_url= ) + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + test -n "$path_url" || ynh_die "ynh_normalize_url_path expect a URL path as first argument and received nothing." if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / path_url="/$path_url" # Add / at begin of path variable @@ -24,12 +29,17 @@ ynh_normalize_url_path () { # Find a free port and return it # -# example: port=$(ynh_find_port 8080) +# example: port=$(ynh_find_port --port=8080) # -# usage: ynh_find_port begin_port -# | arg: begin_port - port to start to search +# usage: ynh_find_port --port=begin_port +# | arg: -p, --port - port to start to search ynh_find_port () { - local port=$1 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [p]=port= ) + local port + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + test -n "$port" || ynh_die "The argument of ynh_find_port must be a valid port." while netcat -z 127.0.0.1 $port # Check if the port is free do @@ -40,28 +50,38 @@ ynh_find_port () { # Check availability of a web path # -# example: ynh_webpath_available some.domain.tld /coffee +# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee # -# usage: ynh_webpath_available domain path -# | arg: domain - the domain/host of the url -# | arg: path - the web path to check the availability of +# usage: ynh_webpath_available --domain=domain --path_url=path +# | arg: -d, --domain - the domain/host of the url +# | arg: -p, --path_url - the web path to check the availability of ynh_webpath_available () { - local domain=$1 - local path=$2 - sudo yunohost domain url-available $domain $path + # Declare an array to define the options of this helper. + declare -Ar args_array=( [d]=domain= [p]=path_url= ) + local domain + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost domain url-available $domain $path_url } # Register/book a web path for an app # -# example: ynh_webpath_register wordpress some.domain.tld /coffee +# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee # -# usage: ynh_webpath_register app domain path -# | arg: app - the app for which the domain should be registered -# | arg: domain - the domain/host of the web path -# | arg: path - the web path to be registered +# usage: ynh_webpath_register --app=app --domain=domain --path_url=path +# | arg: -a, --app - the app for which the domain should be registered +# | arg: -d, --domain - the domain/host of the web path +# | arg: -p, --path_url - the web path to be registered ynh_webpath_register () { - local app=$1 - local domain=$2 - local path=$3 - sudo yunohost app register-url $app $domain $path + # Declare an array to define the options of this helper. + declare -Ar args_array=( [a]=app= [d]=domain= [p]=path_url= ) + local app + local domain + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost app register-url $app $domain $path_url } From 0aed8a692287413af1f38e8adefd684c78a3bccf Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 21 Dec 2018 20:43:57 +0100 Subject: [PATCH 017/113] Use getopts for nodejs helper --- data/helpers.d/nodejs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 5111fa671..d56ef8272 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -53,13 +53,18 @@ ynh_use_nodejs () { # # ynh_install_nodejs will install the version of node provided as argument by using n. # -# usage: ynh_install_nodejs [nodejs_version] -# | arg: nodejs_version - Version of node to install. +# usage: ynh_install_nodejs --nodejs_version=nodejs_version +# | arg: -n, --nodejs_version - Version of node to install. # If possible, prefer to use major version number (e.g. 8 instead of 8.10.0). # The crontab will handle the update of minor versions when needed. ynh_install_nodejs () { # Use n, https://github.com/tj/n to manage the nodejs versions - nodejs_version="$1" + + # Declare an array to define the options of this helper. + declare -Ar args_array=( [n]=nodejs_version= ) + local nodejs_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" # Create $n_install_dir mkdir -p "$n_install_dir" From 0654c68af4c0a14f015d867353960e9f1e7bd4f0 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 21 Dec 2018 20:50:57 +0100 Subject: [PATCH 018/113] Use getopts for package's helpers --- data/helpers.d/package | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/data/helpers.d/package b/data/helpers.d/package index db3b50e0e..6a0f9d1c7 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -1,24 +1,36 @@ # Check either a package is installed or not # -# example: ynh_package_is_installed 'yunohost' && echo "ok" +# example: ynh_package_is_installed --package=yunohost && echo "ok" # -# usage: ynh_package_is_installed name -# | arg: name - the package name to check +# usage: ynh_package_is_installed --package=name +# | arg: -p, --package - the package name to check ynh_package_is_installed() { - dpkg-query -W -f '${Status}' "$1" 2>/dev/null \ + # Declare an array to define the options of this helper. + declare -Ar args_array=( [p]=package= ) + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + dpkg-query -W -f '${Status}' "$package" 2>/dev/null \ | grep -c "ok installed" &>/dev/null } # Get the version of an installed package # -# example: version=$(ynh_package_version 'yunohost') +# example: version=$(ynh_package_version --package=yunohost) # -# usage: ynh_package_version name -# | arg: name - the package name to get version +# usage: ynh_package_version --package=name +# | arg: -p, --package - the package name to get version # | ret: the version or an empty string ynh_package_version() { - if ynh_package_is_installed "$1"; then - dpkg-query -W -f '${Version}' "$1" 2>/dev/null + # Declare an array to define the options of this helper. + declare -Ar args_array=( [p]=package= ) + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_package_is_installed "$package"; then + dpkg-query -W -f '${Version}' "$package" 2>/dev/null else echo '' fi From fe6a414ebf9ef18cb78ccf081f5085ebe15c0362 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 21 Dec 2018 20:57:15 +0100 Subject: [PATCH 019/113] Use getopts for print's helpers --- data/helpers.d/print | 47 +++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/data/helpers.d/print b/data/helpers.d/print index 93d402e64..7d32207f6 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -1,15 +1,28 @@ # Print a message to stderr and exit -# usage: ynh_die MSG [RETCODE] +# usage: ynh_die --message=MSG [--ret_code=RETCODE] ynh_die() { - echo "$1" 1>&2 - exit "${2:-1}" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=message= [c]=ret_code= ) + local message + local ret_code + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + echo "$message" 1>&2 + exit "${ret_code:-1}" } # Display a message in the 'INFO' logging category # -# usage: ynh_info "Some message" +# usage: ynh_info --message="Some message" ynh_print_info() { - echo "$1" >> "$YNH_STDINFO" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=message= ) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + echo "$message" >> "$YNH_STDINFO" } # Ignore the yunohost-cli log to prevent errors with conditional commands @@ -39,18 +52,30 @@ ynh_print_log () { # Print a warning on stderr # -# usage: ynh_print_warn "Text to print" -# | arg: text - The text to print +# usage: ynh_print_warn --message="Text to print" +# | arg: -m, --message - The text to print ynh_print_warn () { - ynh_print_log "\e[93m\e[1m[WARN]\e[0m ${1}" >&2 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=message= ) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_print_log "\e[93m\e[1m[WARN]\e[0m ${message}" >&2 } # Print an error on stderr # -# usage: ynh_print_err "Text to print" -# | arg: text - The text to print +# usage: ynh_print_err --message="Text to print" +# | arg: -m, --message - The text to print ynh_print_err () { - ynh_print_log "\e[91m\e[1m[ERR]\e[0m ${1}" >&2 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=message= ) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_print_log "\e[91m\e[1m[ERR]\e[0m ${message}" >&2 } # Execute a command and print the result as an error From f24b7a6fda2e72660d7db6a4841878cf56551129 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 21 Dec 2018 21:01:43 +0100 Subject: [PATCH 020/113] Use getopts for setting's helpers --- data/helpers.d/setting | 48 ++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/data/helpers.d/setting b/data/helpers.d/setting index ad036ba4f..29eeecd3c 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -1,27 +1,49 @@ # Get an application setting # -# usage: ynh_app_setting_get app key -# | arg: app - the application id -# | arg: key - the setting to get +# usage: ynh_app_setting_get --app=app --key=key +# | arg: -a, --app - the application id +# | arg: -k, --key - the setting to get ynh_app_setting_get() { - sudo yunohost app setting "$1" "$2" --output-as plain --quiet + # Declare an array to define the options of this helper. + declare -Ar args_array=( [a]=app= [k]=key= ) + local app + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost app setting "$app" "$key" --output-as plain --quiet } # Set an application setting # -# usage: ynh_app_setting_set app key value -# | arg: app - the application id -# | arg: key - the setting name to set -# | arg: value - the setting value to set +# usage: ynh_app_setting_set --app=app --key=key --value=value +# | arg: -a, --app - the application id +# | arg: -k, --key - the setting name to set +# | arg: -v, --value - the setting value to set ynh_app_setting_set() { - sudo yunohost app setting "$1" "$2" --value="$3" --quiet + # Declare an array to define the options of this helper. + declare -Ar args_array=( [a]=app= [k]=key= [v]=value= ) + local app + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost app setting "$app" "$key" --value="$value" --quiet } # Delete an application setting # -# usage: ynh_app_setting_delete app key -# | arg: app - the application id -# | arg: key - the setting to delete +# usage: ynh_app_setting_delete --app=app --key=key +# | arg: -a, --app - the application id +# | arg: -k, --key - the setting to delete ynh_app_setting_delete() { - sudo yunohost app setting -d "$1" "$2" --quiet + # Declare an array to define the options of this helper. + declare -Ar args_array=( [a]=app= [k]=key= ) + local app + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost app setting -d "$app" "$key" --quiet } From c057d38fe136833218d06058374f3b11544b8224 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 17:58:22 +0100 Subject: [PATCH 021/113] Use getopts for string's helpers --- data/helpers.d/string | 59 +++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/data/helpers.d/string b/data/helpers.d/string index f708b31b1..c13497f9f 100644 --- a/data/helpers.d/string +++ b/data/helpers.d/string @@ -1,59 +1,74 @@ # Generate a random string # -# example: pwd=$(ynh_string_random 8) +# example: pwd=$(ynh_string_random --length=8) # -# usage: ynh_string_random [length] -# | arg: length - the string length to generate (default: 24) +# usage: ynh_string_random [--length=string_length] +# | arg: -l, --length - the string length to generate (default: 24) ynh_string_random() { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [l]=length= ) + local length + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + length=${length:-24} + dd if=/dev/urandom bs=1 count=200 2> /dev/null \ | tr -c -d 'A-Za-z0-9' \ - | sed -n 's/\(.\{'"${1:-24}"'\}\).*/\1/p' + | sed -n 's/\(.\{'"$length"'\}\).*/\1/p' } # Substitute/replace a string (or expression) by another in a file # # usage: ynh_replace_string match_string replace_string target_file -# | arg: match_string - String to be searched and replaced in the file -# | arg: replace_string - String that will replace matches -# | arg: target_file - File in which the string will be replaced. +# | arg: -m, --match_string - String to be searched and replaced in the file +# | arg: -r, --replace_string - String that will replace matches +# | arg: -f, --target_file - File in which the string will be replaced. # # As this helper is based on sed command, regular expressions and # references to sub-expressions can be used # (see sed manual page for more information) ynh_replace_string () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= ) + local match_string + local replace_string + local target_file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local delimit=@ - local match_string=$1 - local replace_string=$2 - local workfile=$3 - # Escape the delimiter if it's in the string. match_string=${match_string//${delimit}/"\\${delimit}"} replace_string=${replace_string//${delimit}/"\\${delimit}"} - sudo sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$workfile" + sudo sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file" } # Substitute/replace a special string by another in a file # # usage: ynh_replace_special_string match_string replace_string target_file -# | arg: match_string - String to be searched and replaced in the file -# | arg: replace_string - String that will replace matches -# | arg: target_file - File in which the string will be replaced. +# | arg: -m, --match_string - String to be searched and replaced in the file +# | arg: -r, --replace_string - String that will replace matches +# | arg: -t, --target_file - File in which the string will be replaced. # # This helper will use ynh_replace_string, but as you can use special # characters, you can't use some regular expressions and sub-expressions. ynh_replace_special_string () { - local match_string=$1 - local replace_string=$2 - local workfile=$3 + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= ) + local match_string + local replace_string + local target_file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" - # Escape any backslash to preserve them as simple backslash. - match_string=${match_string//\\/"\\\\"} - replace_string=${replace_string//\\/"\\\\"} + # Escape any backslash to preserve them as simple backslash. + match_string=${match_string//\\/"\\\\"} + replace_string=${replace_string//\\/"\\\\"} # Escape the & character, who has a special function in sed. match_string=${match_string//&/"\&"} replace_string=${replace_string//&/"\&"} - ynh_replace_string "$match_string" "$replace_string" "$workfile" + ynh_replace_string "$match_string" "$replace_string" "$target_file" } From 6425117c4dcaa7f498dc91d22ddf13b1c388e64e Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:12:33 +0100 Subject: [PATCH 022/113] Use getopts for user's helpers --- data/helpers.d/user | 76 ++++++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/data/helpers.d/user b/data/helpers.d/user index 47e6eb88a..4a2e7c55d 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -2,22 +2,35 @@ # # example: ynh_user_exists 'toto' || exit 1 # -# usage: ynh_user_exists username -# | arg: username - the username to check +# usage: ynh_user_exists --username=username +# | arg: -u, --username - the username to check ynh_user_exists() { - sudo yunohost user list --output-as json | grep -q "\"username\": \"${1}\"" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=username= ) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost user list --output-as json | grep -q "\"username\": \"${username}\"" } # Retrieve a YunoHost user information # # example: mail=$(ynh_user_get_info 'toto' 'mail') # -# usage: ynh_user_get_info username key -# | arg: username - the username to retrieve info from -# | arg: key - the key to retrieve +# usage: ynh_user_get_info --username=username --key=key +# | arg: -u, --username - the username to retrieve info from +# | arg: -k, --key - the key to retrieve # | ret: string - the key's value ynh_user_get_info() { - sudo yunohost user info "$1" --output-as plain | ynh_get_plain_key "$2" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=username= [k]=key= ) + local username + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost user info "$username" --output-as plain | ynh_get_plain_key "$key" } # Get the list of YunoHost users @@ -33,39 +46,58 @@ ynh_user_list() { # Check if a user exists on the system # -# usage: ynh_system_user_exists username -# | arg: username - the username to check +# usage: ynh_system_user_exists --username=username +# | arg: -u, --username - the username to check ynh_system_user_exists() { - getent passwd "$1" &>/dev/null + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=username= ) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent passwd "$username" &>/dev/null } # Create a system user # -# usage: ynh_system_user_create user_name [home_dir] -# | arg: user_name - Name of the system user that will be create -# | arg: home_dir - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home +# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] +# | arg: -u, --username - Name of the system user that will be create +# | arg: -h, --home_dir - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home ynh_system_user_create () { - if ! ynh_system_user_exists "$1" # Check if the user exists on the system + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=username= [h]=home_dir= ) + local username + local home_dir + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! ynh_system_user_exists "$username" # Check if the user exists on the system then # If the user doesn't exist if [ $# -ge 2 ]; then # If a home dir is mentioned - local user_home_dir="-d $2" + local user_home_dir="-d $home_dir" else local user_home_dir="--no-create-home" fi - sudo useradd $user_home_dir --system --user-group $1 --shell /usr/sbin/nologin || ynh_die "Unable to create $1 system account" + sudo useradd $user_home_dir --system --user-group $username --shell /usr/sbin/nologin || ynh_die "Unable to create $username system account" fi } # Delete a system user # -# usage: ynh_system_user_delete user_name -# | arg: user_name - Name of the system user that will be create +# usage: ynh_system_user_delete --username=user_name +# | arg: -u, --username - Name of the system user that will be create ynh_system_user_delete () { - if ynh_system_user_exists "$1" # Check if the user exists on the system + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=username= ) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_system_user_exists "$username" # Check if the user exists on the system then - echo "Remove the user $1" >&2 - sudo userdel $1 + echo "Remove the user $username" >&2 + sudo userdel $username else - echo "The user $1 was not found" >&2 + echo "The user $username was not found" >&2 fi } From bc08c5c872dda353d3079a010c7fb18a6c0c845a Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:17:08 +0100 Subject: [PATCH 023/113] Use getopts for ynh_setup_source helper --- data/helpers.d/utils | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 07b8a6d96..339d51256 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -141,22 +141,27 @@ ynh_backup_before_upgrade () { # sources/extra_files/$src_id will be applied to dest_dir # # -# usage: ynh_setup_source dest_dir [source_id] -# | arg: dest_dir - Directory where to setup sources -# | arg: source_id - Name of the app, if the package contains more than one app +# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] +# | arg: -d, --dest_dir - Directory where to setup sources +# | arg: -s, --source_id - Name of the app, if the package contains more than one app ynh_setup_source () { - local dest_dir=$1 - local src_id=${2:-app} # If the argument is not given, source_id equals "app" + # Declare an array to define the options of this helper. + declare -Ar args_array=( [d]=dest_dir= [s]=source_id= ) + local dest_dir + local source_id + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + source_id="${source_id:-app}" # If the argument is not given, source_id equals "app" # Load value from configuration file (see above for a small doc about this file # format) - local src_url=$(grep 'SOURCE_URL=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-) - local src_sum=$(grep 'SOURCE_SUM=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-) - local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-) - local src_format=$(grep 'SOURCE_FORMAT=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-) - local src_extract=$(grep 'SOURCE_EXTRACT=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-) - local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-) - local src_filename=$(grep 'SOURCE_FILENAME=' "$YNH_CWD/../conf/${src_id}.src" | cut -d= -f2-) + local src_url=$(grep 'SOURCE_URL=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_sum=$(grep 'SOURCE_SUM=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_filename=$(grep 'SOURCE_FILENAME=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) # Default value src_sumprg=${src_sumprg:-sha256sum} @@ -165,7 +170,7 @@ ynh_setup_source () { src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') src_extract=${src_extract:-true} if [ "$src_filename" = "" ] ; then - src_filename="${src_id}.${src_format}" + src_filename="${source_id}.${src_format}" fi local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}" @@ -211,18 +216,18 @@ ynh_setup_source () { fi # Apply patches - if (( $(find $YNH_CWD/../sources/patches/ -type f -name "${src_id}-*.patch" 2> /dev/null | wc -l) > "0" )); then + if (( $(find $YNH_CWD/../sources/patches/ -type f -name "${source_id}-*.patch" 2> /dev/null | wc -l) > "0" )); then local old_dir=$(pwd) (cd "$dest_dir" \ - && for p in $YNH_CWD/../sources/patches/${src_id}-*.patch; do \ + && for p in $YNH_CWD/../sources/patches/${source_id}-*.patch; do \ patch -p1 < $p; done) \ || ynh_die "Unable to apply patches" cd $old_dir fi # Add supplementary files - if test -e "$YNH_CWD/../sources/extra_files/${src_id}"; then - cp -a $YNH_CWD/../sources/extra_files/$src_id/. "$dest_dir" + if test -e "$YNH_CWD/../sources/extra_files/${source_id}"; then + cp -a $YNH_CWD/../sources/extra_files/$source_id/. "$dest_dir" fi } From d3a501aa416221c91ebc1154bef572e682e39c52 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:50:54 +0100 Subject: [PATCH 024/113] Use getopts helpers in string --- data/helpers.d/string | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/string b/data/helpers.d/string index c13497f9f..188c2da62 100644 --- a/data/helpers.d/string +++ b/data/helpers.d/string @@ -19,7 +19,7 @@ ynh_string_random() { # Substitute/replace a string (or expression) by another in a file # -# usage: ynh_replace_string match_string replace_string target_file +# usage: ynh_replace_string --match_string=match_string --replace_string=replace_string --target_file=target_file # | arg: -m, --match_string - String to be searched and replaced in the file # | arg: -r, --replace_string - String that will replace matches # | arg: -f, --target_file - File in which the string will be replaced. @@ -46,7 +46,7 @@ ynh_replace_string () { # Substitute/replace a special string by another in a file # -# usage: ynh_replace_special_string match_string replace_string target_file +# usage: ynh_replace_special_string --match_string=match_string --replace_string=replace_string --target_file=target_file # | arg: -m, --match_string - String to be searched and replaced in the file # | arg: -r, --replace_string - String that will replace matches # | arg: -t, --target_file - File in which the string will be replaced. @@ -70,5 +70,5 @@ ynh_replace_special_string () { match_string=${match_string//&/"\&"} replace_string=${replace_string//&/"\&"} - ynh_replace_string "$match_string" "$replace_string" "$target_file" + ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file" } From 8c600e1e0ba2139a60854c8acbf3b11c59362a8e Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:51:12 +0100 Subject: [PATCH 025/113] Use getopts helpers in mysql --- data/helpers.d/mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 0cfe65380..4898bed92 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -176,7 +176,7 @@ ynh_mysql_setup_db () { db_password="${db_password:-$new_db_pwd}" ynh_mysql_create_db "$db_name" "$db_user" "$db_password" # Create the database - ynh_app_setting_set $app mysqlpwd $db_password # Store the password in the app's config + ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_password # Store the password in the app's config } # Remove a database if it exists, and the associated user From 43e4a34d212737161bcc7a127f32bba7fd6ff697 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:51:21 +0100 Subject: [PATCH 026/113] Use getopts helpers in network --- data/helpers.d/network | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/network b/data/helpers.d/network index fc7eb3f69..0059702e9 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -17,7 +17,7 @@ ynh_normalize_url_path () { # Manage arguments with getopts ynh_handle_getopts_args "$@" - test -n "$path_url" || ynh_die "ynh_normalize_url_path expect a URL path as first argument and received nothing." + test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / path_url="/$path_url" # Add / at begin of path variable fi @@ -40,7 +40,7 @@ ynh_find_port () { # Manage arguments with getopts ynh_handle_getopts_args "$@" - test -n "$port" || ynh_die "The argument of ynh_find_port must be a valid port." + test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." while netcat -z 127.0.0.1 $port # Check if the port is free do port=$((port+1)) # Else, pass to next port From 68d6adf8100a43e7e045746e9045a13fc03e0207 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:51:34 +0100 Subject: [PATCH 027/113] Use getopts helpers in package --- data/helpers.d/package | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/package b/data/helpers.d/package index 6a0f9d1c7..cb5de5f06 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -121,7 +121,7 @@ ynh_package_install_from_equivs () { && equivs-build ./control 1>/dev/null \ && sudo dpkg --force-depends \ -i "./${pkgname}_${pkgversion}_all.deb" 2>&1 \ - && ynh_package_install -f) || ynh_die "Unable to install dependencies" + && ynh_package_install -f) || ynh_die --message="Unable to install dependencies" [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR # Remove the temp dir. # check if the package is actually installed @@ -162,9 +162,9 @@ Description: Fake package for ${app} (YunoHost app) dependencies This meta-package is only responsible of installing its dependencies. EOF ynh_package_install_from_equivs /tmp/${dep_app}-ynh-deps.control \ - || ynh_die "Unable to install dependencies" # Install the fake package and its dependencies + || ynh_die --message="Unable to install dependencies" # Install the fake package and its dependencies rm /tmp/${dep_app}-ynh-deps.control - ynh_app_setting_set $app apt_dependencies $dependencies + ynh_app_setting_set --app=$app --key=apt_dependencies --value="$dependencies" } # Remove fake package and its dependencies From 0aa8a2ede354d4c1cf03576549d0aeebe3e6cdb8 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:51:39 +0100 Subject: [PATCH 028/113] Use getopts helpers in user --- data/helpers.d/user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/user b/data/helpers.d/user index 4a2e7c55d..3947ebe7f 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -78,7 +78,7 @@ ynh_system_user_create () { else local user_home_dir="--no-create-home" fi - sudo useradd $user_home_dir --system --user-group $username --shell /usr/sbin/nologin || ynh_die "Unable to create $username system account" + sudo useradd $user_home_dir --system --user-group $username --shell /usr/sbin/nologin || ynh_die --message="Unable to create $username system account" fi } From 2094557e1e32f7b8d56406d4229fcd8e1f04f1fd Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:51:49 +0100 Subject: [PATCH 029/113] Use getopts helpers in getopts itself --- data/helpers.d/getopts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/getopts b/data/helpers.d/getopts index 1cc66da8a..827e08477 100644 --- a/data/helpers.d/getopts +++ b/data/helpers.d/getopts @@ -98,10 +98,10 @@ ynh_handle_getopts_args () { if [ "$parameter" = "?" ] then - ynh_die "Invalid argument: -${OPTARG:-}" + ynh_die --message="Invalid argument: -${OPTARG:-}" elif [ "$parameter" = ":" ] then - ynh_die "-$OPTARG parameter requires an argument." + ynh_die --message="-$OPTARG parameter requires an argument." else local shift_value=1 # Use the long option, corresponding to the short option read by getopts, as a variable From 2c6b76e099a2e1f81feb2926dee869472d3e2b6f Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:52:03 +0100 Subject: [PATCH 030/113] Use getopts helpers in nodejs --- data/helpers.d/nodejs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index d56ef8272..fcba2d75a 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -15,7 +15,7 @@ ynh_install_n () { echo "SOURCE_URL=https://github.com/tj/n/archive/v2.1.7.tar.gz SOURCE_SUM=2ba3c9d4dd3c7e38885b37e02337906a1ee91febe6d5c9159d89a9050f2eea8f" > "../conf/n.src" # Download and extract n - ynh_setup_source "$n_install_dir/git" n + ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n # Install n (cd "$n_install_dir/git" PREFIX=$N_PREFIX make install 2>&1) @@ -35,7 +35,7 @@ SOURCE_SUM=2ba3c9d4dd3c7e38885b37e02337906a1ee91febe6d5c9159d89a9050f2eea8f" > " # # usage: ynh_use_nodejs ynh_use_nodejs () { - nodejs_version=$(ynh_app_setting_get $app nodejs_version) + nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version) nodejs_use_version="echo \"Deprecated command, should be removed\"" @@ -85,7 +85,7 @@ ynh_install_nodejs () { fi # Modify the default N_PREFIX in n script - ynh_replace_string "^N_PREFIX=\${N_PREFIX-.*}$" "N_PREFIX=\${N_PREFIX-$N_PREFIX}" "$n_install_dir/bin/n" + ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH @@ -111,7 +111,7 @@ ynh_install_nodejs () { echo "$YNH_APP_ID:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version" # Store nodejs_version into the config of this app - ynh_app_setting_set $app nodejs_version $nodejs_version + ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version # Build the update script and set the cronjob ynh_cron_upgrade_node @@ -127,7 +127,7 @@ ynh_install_nodejs () { # # usage: ynh_remove_nodejs ynh_remove_nodejs () { - nodejs_version=$(ynh_app_setting_get $app nodejs_version) + nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version) # Remove the line for this app sed --in-place "/$YNH_APP_ID:$nodejs_version/d" "$n_install_dir/ynh_app_version" @@ -141,8 +141,8 @@ ynh_remove_nodejs () { # If no other app uses n, remove n if [ ! -s "$n_install_dir/ynh_app_version" ] then - ynh_secure_remove "$n_install_dir" - ynh_secure_remove "/usr/local/n" + ynh_secure_remove --file="$n_install_dir" + ynh_secure_remove --file="/usr/local/n" sed --in-place "/N_PREFIX/d" /root/.bashrc rm -f /etc/cron.daily/node_update fi From 80c0b7bdf2261b3476217ce70b9618f850a65727 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:52:12 +0100 Subject: [PATCH 031/113] Use getopts helpers in utils --- data/helpers.d/utils | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 339d51256..b6d7c11f6 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -49,7 +49,7 @@ ynh_restore_upgradebackup () { sudo yunohost app remove $app # Restore the backup sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force - ynh_die "The app was restored to the way it was before the failed upgrade." + ynh_die --message="The app was restored to the way it was before the failed upgrade." fi else echo "\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" >&2 @@ -97,7 +97,7 @@ ynh_backup_before_upgrade () { sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null fi else - ynh_die "Backup failed, the upgrade process was aborted." + ynh_die --message="Backup failed, the upgrade process was aborted." fi else echo "\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" @@ -178,12 +178,12 @@ ynh_setup_source () { then # Use the local source file if it is present cp $local_src $src_filename else # If not, download the source - local out=`wget -nv -O $src_filename $src_url 2>&1` || ynh_print_err $out + local out=`wget -nv -O $src_filename $src_url 2>&1` || ynh_print_err --message="$out" fi # Check the control sum echo "${src_sum} ${src_filename}" | ${src_sumprg} -c --status \ - || ynh_die "Corrupt source" + || ynh_die --message="Corrupt source" # Extract source into the app dir mkdir -p "$dest_dir" @@ -199,7 +199,7 @@ ynh_setup_source () { local tmp_dir=$(mktemp -d) unzip -quo $src_filename -d "$tmp_dir" cp -a $tmp_dir/*/. "$dest_dir" - ynh_secure_remove "$tmp_dir" + ynh_secure_remove --file="$tmp_dir" else unzip -quo $src_filename -d "$dest_dir" fi @@ -211,7 +211,7 @@ ynh_setup_source () { if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]] ; then tar -xf $src_filename -C "$dest_dir" $strip else - ynh_die "Archive format unrecognized." + ynh_die --message="Archive format unrecognized." fi fi @@ -221,7 +221,7 @@ ynh_setup_source () { (cd "$dest_dir" \ && for p in $YNH_CWD/../sources/patches/${source_id}-*.patch; do \ patch -p1 < $p; done) \ - || ynh_die "Unable to apply patches" + || ynh_die --message="Unable to apply patches" cd $old_dir fi From 705fe435ed43a26048a9348e3f86891e701db069 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:52:23 +0100 Subject: [PATCH 032/113] Use getopts helpers in backend --- data/helpers.d/backend | 60 +++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 025ce45f0..840fcfd6a 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -122,18 +122,18 @@ ynh_add_systemd_config () { local template="${nonappend:-systemd.service}" finalsystemdconf="/etc/systemd/system/$service.service" - ynh_backup_if_checksum_is_different "$finalsystemdconf" + ynh_backup_if_checksum_is_different --file="$finalsystemdconf" sudo cp ../conf/$template "$finalsystemdconf" # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. # Substitute in a nginx config file only if the variable is not empty if test -n "${final_path:-}"; then - ynh_replace_string "__FINALPATH__" "$final_path" "$finalsystemdconf" + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalsystemdconf" fi if test -n "${app:-}"; then - ynh_replace_string "__APP__" "$app" "$finalsystemdconf" + ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finalsystemdconf" fi - ynh_store_file_checksum "$finalsystemdconf" + ynh_store_file_checksum --file="$finalsystemdconf" sudo chown root: "$finalsystemdconf" sudo systemctl enable $service @@ -157,7 +157,7 @@ ynh_remove_systemd_config () { if [ -e "$finalsystemdconf" ]; then sudo systemctl stop $service sudo systemctl disable $service - ynh_secure_remove "$finalsystemdconf" + ynh_secure_remove --file="$finalsystemdconf" sudo systemctl daemon-reload fi } @@ -183,7 +183,7 @@ ynh_remove_systemd_config () { ynh_add_nginx_config () { finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf" local others_var=${1:-} - ynh_backup_if_checksum_is_different "$finalnginxconf" + ynh_backup_if_checksum_is_different --file="$finalnginxconf" sudo cp ../conf/nginx.conf "$finalnginxconf" # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. @@ -191,20 +191,20 @@ ynh_add_nginx_config () { if test -n "${path_url:-}"; then # path_url_slash_less is path_url, or a blank value if path_url is only '/' local path_url_slash_less=${path_url%/} - ynh_replace_string "__PATH__/" "$path_url_slash_less/" "$finalnginxconf" - ynh_replace_string "__PATH__" "$path_url" "$finalnginxconf" + ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$finalnginxconf" + ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finalnginxconf" fi if test -n "${domain:-}"; then - ynh_replace_string "__DOMAIN__" "$domain" "$finalnginxconf" + ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$finalnginxconf" fi if test -n "${port:-}"; then - ynh_replace_string "__PORT__" "$port" "$finalnginxconf" + ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$finalnginxconf" fi if test -n "${app:-}"; then - ynh_replace_string "__NAME__" "$app" "$finalnginxconf" + ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$finalnginxconf" fi if test -n "${final_path:-}"; then - ynh_replace_string "__FINALPATH__" "$final_path" "$finalnginxconf" + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalnginxconf" fi # Replace all other variable given as arguments @@ -212,17 +212,17 @@ ynh_add_nginx_config () { do # ${var_to_replace^^} make the content of the variable on upper-cases # ${!var_to_replace} get the content of the variable named $var_to_replace - ynh_replace_string "__${var_to_replace^^}__" "${!var_to_replace}" "$finalnginxconf" + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalnginxconf" done if [ "${path_url:-}" != "/" ] then - ynh_replace_string "^#sub_path_only" "" "$finalnginxconf" + ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf" else - ynh_replace_string "^#root_path_only" "" "$finalnginxconf" + ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf" fi - ynh_store_file_checksum "$finalnginxconf" + ynh_store_file_checksum --file="$finalnginxconf" sudo systemctl reload nginx } @@ -231,7 +231,7 @@ ynh_add_nginx_config () { # # usage: ynh_remove_nginx_config ynh_remove_nginx_config () { - ynh_secure_remove "/etc/nginx/conf.d/$domain.d/$app.conf" + ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf" sudo systemctl reload nginx } @@ -247,24 +247,24 @@ ynh_add_fpm_config () { fpm_config_dir="/etc/php5/fpm" fpm_service="php5-fpm" fi - ynh_app_setting_set $app fpm_config_dir "$fpm_config_dir" - ynh_app_setting_set $app fpm_service "$fpm_service" + ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" + ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" finalphpconf="$fpm_config_dir/pool.d/$app.conf" - ynh_backup_if_checksum_is_different "$finalphpconf" + ynh_backup_if_checksum_is_different --file="$finalphpconf" sudo cp ../conf/php-fpm.conf "$finalphpconf" - ynh_replace_string "__NAMETOCHANGE__" "$app" "$finalphpconf" - ynh_replace_string "__FINALPATH__" "$final_path" "$finalphpconf" - ynh_replace_string "__USER__" "$app" "$finalphpconf" + ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf" + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf" + ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf" sudo chown root: "$finalphpconf" - ynh_store_file_checksum "$finalphpconf" + ynh_store_file_checksum --file="$finalphpconf" if [ -e "../conf/php-fpm.ini" ] then finalphpini="$fpm_config_dir/conf.d/20-$app.ini" - ynh_backup_if_checksum_is_different "$finalphpini" + ynh_backup_if_checksum_is_different --file="$finalphpini" sudo cp ../conf/php-fpm.ini "$finalphpini" sudo chown root: "$finalphpini" - ynh_store_file_checksum "$finalphpini" + ynh_store_file_checksum --file="$finalphpini" fi sudo systemctl reload $fpm_service } @@ -273,14 +273,14 @@ ynh_add_fpm_config () { # # usage: ynh_remove_fpm_config ynh_remove_fpm_config () { - local fpm_config_dir=$(ynh_app_setting_get $app fpm_config_dir) - local fpm_service=$(ynh_app_setting_get $app fpm_service) + local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) # Assume php version 5 if not set if [ -z "$fpm_config_dir" ]; then fpm_config_dir="/etc/php5/fpm" fpm_service="php5-fpm" fi - ynh_secure_remove "$fpm_config_dir/pool.d/$app.conf" - ynh_secure_remove "$fpm_config_dir/conf.d/20-$app.ini" 2>&1 + ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" + ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1 sudo systemctl reload $fpm_service } From e3bebf017858a5c979fa7121e396c20fe0c3a130 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 29 Dec 2018 18:52:39 +0100 Subject: [PATCH 033/113] Use getopts helpers in filesystem --- data/helpers.d/filesystem | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 34b0487cc..4cb60f7cb 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -145,7 +145,7 @@ ynh_restore () { while read line; do local ORIGIN_PATH=$(echo "$line" | grep -ohP "^\"\K.*(?=\",\".*\"$)") local ARCHIVE_PATH=$(echo "$line" | grep -ohP "^\".*\",\"$REL_DIR\K.*(?=\"$)") - ynh_restore_file "$ARCHIVE_PATH" "$ORIGIN_PATH" + ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH" done } @@ -223,7 +223,7 @@ ynh_restore_file () { mkdir -p "$(dirname "$backup_file")" mv "${dest_path}" "$backup_file" # Move the current file or directory else - ynh_secure_remove ${dest_path} + ynh_secure_remove --file=${dest_path} fi fi @@ -289,7 +289,7 @@ ynh_store_file_checksum () { ynh_handle_getopts_args "$@" local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - ynh_app_setting_set $app $checksum_setting_name $(sudo md5sum "$file" | cut -d' ' -f1) + ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1) } # Verify the checksum and backup the file if it's different @@ -310,7 +310,7 @@ ynh_backup_if_checksum_is_different () { ynh_handle_getopts_args "$@" local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - local checksum_value=$(ynh_app_setting_get $app $checksum_setting_name) + local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) if [ -n "$checksum_value" ] then # Proceed only if a value was stored into the app settings if ! echo "$checksum_value $file" | sudo md5sum -c --status @@ -338,12 +338,12 @@ ynh_delete_file_checksum () { ynh_handle_getopts_args "$@" local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - ynh_app_setting_delete $app $checksum_setting_name + ynh_app_setting_delete --app=$app --key=$checksum_setting_name } # Remove a file or a directory securely # -# usage: ynh_secure_remove file=path_to_remove +# usage: ynh_secure_remove --file=path_to_remove # | arg: -f, --file - File or directory to remove ynh_secure_remove () { # Declare an array to define the options of this helper. From 976f160afbf34be27b04cd0bc1dc13870d642848 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 4 Jan 2019 17:11:49 +0100 Subject: [PATCH 034/113] Fix getopts and its legacy mode --- data/helpers.d/backend | 3 +++ data/helpers.d/filesystem | 8 ++++++++ data/helpers.d/getopts | 42 +++++++++++++++++++++------------------ data/helpers.d/ip | 5 +++++ data/helpers.d/mysql | 10 ++++++++++ data/helpers.d/network | 6 ++++++ data/helpers.d/nodejs | 3 +++ data/helpers.d/package | 4 ++++ data/helpers.d/print | 6 ++++++ data/helpers.d/setting | 5 +++++ data/helpers.d/string | 5 +++++ data/helpers.d/system | 2 ++ data/helpers.d/user | 7 +++++++ data/helpers.d/utils | 3 +++ 14 files changed, 90 insertions(+), 19 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 12e2b1964..2109b17ca 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -16,6 +16,7 @@ # Unless you use the option --non-append ynh_use_logrotate () { # Declare an array to define the options of this helper. + local legacy_args=lnuya declare -Ar args_array=( [l]=logfile= [n]=nonappend [u]=specific_user= [y]=non [a]=append ) # [y]=non [a]=append are only for legacy purpose, to not fail on the old option '--non-append' local logfile @@ -113,6 +114,7 @@ ynh_remove_logrotate () { # ynh_add_systemd_config () { # Declare an array to define the options of this helper. + local legacy_args=st declare -Ar args_array=( [s]=service= [t]=template= ) local service local template @@ -147,6 +149,7 @@ ynh_add_systemd_config () { # ynh_remove_systemd_config () { # Declare an array to define the options of this helper. + local legacy_args=s declare -Ar args_array=( [s]=service= ) local service # Manage arguments with getopts diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 55bad87e8..2e9f93166 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -1,3 +1,5 @@ +#!/bin/bash + CAN_BIND=${CAN_BIND:-1} # Add a file or a directory to the list of paths to backup @@ -46,6 +48,7 @@ ynh_backup() { # TODO find a way to avoid injection by file strange naming ! # Declare an array to define the options of this helper. + local legacy_args=sdbm declare -Ar args_array=( [s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory ) local src_path local dest_path @@ -214,6 +217,7 @@ with open(sys.argv[1], 'r') as backup_file: # ynh_restore_file () { # Declare an array to define the options of this helper. + local legacy_args=odm declare -Ar args_array=( [o]=origin_path= [d]=dest_path= [m]=not_mandatory ) local origin_path local archive_path @@ -307,6 +311,7 @@ properly with chmod/chown." >&2 # | arg: -f, --file - The file on which the checksum will performed, then stored. ynh_store_file_checksum () { # Declare an array to define the options of this helper. + local legacy_args=f declare -Ar args_array=( [f]=file= ) local file # Manage arguments with getopts @@ -328,6 +333,7 @@ ynh_store_file_checksum () { # | ret: Return the name a the backup file, or nothing ynh_backup_if_checksum_is_different () { # Declare an array to define the options of this helper. + local legacy_args=f declare -Ar args_array=( [f]=file= ) local file # Manage arguments with getopts @@ -356,6 +362,7 @@ ynh_backup_if_checksum_is_different () { # | arg: -f, --file= - The file for which the checksum will be deleted ynh_delete_file_checksum () { # Declare an array to define the options of this helper. + local legacy_args=f declare -Ar args_array=( [f]=file= ) local file # Manage arguments with getopts @@ -371,6 +378,7 @@ ynh_delete_file_checksum () { # | arg: -f, --file - File or directory to remove ynh_secure_remove () { # Declare an array to define the options of this helper. + local legacy_args=f declare -Ar args_array=( [f]=file= ) local file # Manage arguments with getopts diff --git a/data/helpers.d/getopts b/data/helpers.d/getopts index b9897f7d9..e9fbe18e8 100644 --- a/data/helpers.d/getopts +++ b/data/helpers.d/getopts @@ -53,33 +53,33 @@ ynh_handle_getopts_args () { # For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u) # And built parameters string for getopts - # ${!args_array[@]} is the list of all keys in the array (A key is 'u' in [u]=user, user is a value) + # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value) local getopts_parameters="" - local key="" - for key in "${!args_array[@]}" + local option_flag="" + for option_flag in "${!args_array[@]}" do - # Concatenate each keys of the array to build the string of arguments for getopts + # Concatenate each option_flags of the array to build the string of arguments for getopts # Will looks like 'abcd' for -a -b -c -d - # If the value of a key finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob) - # Check the last character of the value associate to the key - if [ "${args_array[$key]: -1}" = "=" ] + # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob) + # Check the last character of the value associate to the option_flag + if [ "${args_array[$option_flag]: -1}" = "=" ] then # For an option with additionnal values, add a ':' after the letter for getopts. - getopts_parameters="${getopts_parameters}${key}:" + getopts_parameters="${getopts_parameters}${option_flag}:" else - getopts_parameters="${getopts_parameters}${key}" + getopts_parameters="${getopts_parameters}${option_flag}" fi # Check each argument given to the function local arg="" # ${#arguments[@]} is the size of the array for arg in `seq 0 $(( ${#arguments[@]} - 1 ))` do - # And replace long option (value of the key) by the short option, the key itself + # And replace long option (value of the option_flag) by the short option, the option_flag itself # (e.g. for [u]=user, --user will be -u) # Replace long option with = - arguments[arg]="${arguments[arg]//--${args_array[$key]}/-${key} }" + arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}/-${option_flag} }" # And long option without = - arguments[arg]="${arguments[arg]//--${args_array[$key]%=}/-${key}}" + arguments[arg]="${arguments[arg]//--${args_array[$option_flag]%=}/-${option_flag}}" done done @@ -132,6 +132,7 @@ ynh_handle_getopts_args () { # Declare the content of option_var as a variable. eval ${option_var}="" # Then read the array value per value + local i for i in `seq 0 $(( ${#all_args[@]} - 1 ))` do # If this argument is an option, end here. @@ -166,24 +167,27 @@ ynh_handle_getopts_args () { if [ "${arguments[0]:0:1}" != "-" ] then # If not, enter in legacy mode and manage the arguments as positionnal ones. - echo "! Helper used in legacy mode !" + ynh_print_info --message="! Helper used in legacy mode !" + local i for i in `seq 0 $(( ${#arguments[@]} -1 ))` do - # Use getopts_parameters as a list of key of the array args_array + # Try to use legacy_args as a list of option_flag of the array args_array + # Otherwise, fallback to getopts_parameters to get the option_flag. But an associative arrays isn't always sorted in the correct order... # Remove all ':' in getopts_parameters - getopts_parameters=${getopts_parameters//:} - # Get the key from getopts_parameters, by using the key according to the position of the argument. - key=${getopts_parameters:$i:1} - # Use the long option, corresponding to the key, as a variable + getopts_parameters=${legacy_args:-${getopts_parameters//:}} + # Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument. + option_flag=${getopts_parameters:$i:1} + # Use the long option, corresponding to the option_flag, as a variable # (e.g. for [u]=user, 'user' will be used as a variable) # Also, remove '=' at the end of the long option # The variable name will be stored in 'option_var' - local option_var="${args_array[$key]%=}" + local option_var="${args_array[$option_flag]%=}" # Store each value given as argument in the corresponding variable # The values will be stored in the same order than $args_array eval ${option_var}+=\"${arguments[$i]}\" done + unset legacy_args else # END LEGACY MODE # Call parse_arg and pass the modified list of args as an array of arguments. diff --git a/data/helpers.d/ip b/data/helpers.d/ip index 09c974782..c50d8be73 100644 --- a/data/helpers.d/ip +++ b/data/helpers.d/ip @@ -1,3 +1,5 @@ +#!/bin/bash + # Validate an IP address # # usage: ynh_validate_ip --family=family --ip_address=ip_address @@ -10,6 +12,7 @@ ynh_validate_ip() # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 # Declare an array to define the options of this helper. + local legacy_args=fi declare -Ar args_array=( [f]=family= [i]=ip_address= ) local family local ip_address @@ -40,6 +43,7 @@ EOF ynh_validate_ip4() { # Declare an array to define the options of this helper. + local legacy_args=i declare -Ar args_array=( [i]=ip_address= ) local ip_address # Manage arguments with getopts @@ -59,6 +63,7 @@ ynh_validate_ip4() ynh_validate_ip6() { # Declare an array to define the options of this helper. + local legacy_args=i declare -Ar args_array=( [i]=ip_address= ) local ip_address # Manage arguments with getopts diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 4898bed92..538b00469 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -1,3 +1,5 @@ +#!/bin/bash + MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql # Open a connection as a user @@ -11,6 +13,7 @@ MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql # | arg: -d, --database - the database to connect to ynh_mysql_connect_as() { # Declare an array to define the options of this helper. + local legacy_args=upd declare -Ar args_array=( [u]=user= [p]=password= [d]=database= ) local user local password @@ -29,6 +32,7 @@ ynh_mysql_connect_as() { # | arg: -d, --database - the database to connect to ynh_mysql_execute_as_root() { # Declare an array to define the options of this helper. + local legacy_args=sd declare -Ar args_array=( [s]=sql= [d]=database= ) local sql local database @@ -47,6 +51,7 @@ ynh_mysql_execute_as_root() { # | arg: -d, --database - the database to connect to ynh_mysql_execute_file_as_root() { # Declare an array to define the options of this helper. + local legacy_args=fd declare -Ar args_array=( [f]=file= [d]=database= ) local file local database @@ -103,6 +108,7 @@ ynh_mysql_drop_db() { # | ret: the mysqldump output ynh_mysql_dump_db() { # Declare an array to define the options of this helper. + local legacy_args=d declare -Ar args_array=( [d]=database= ) local database # Manage arguments with getopts @@ -130,6 +136,7 @@ ynh_mysql_create_user() { ynh_mysql_user_exists() { # Declare an array to define the options of this helper. + local legacy_args=u declare -Ar args_array=( [u]=user= ) local user # Manage arguments with getopts @@ -164,6 +171,7 @@ ynh_mysql_drop_user() { # | arg: -p, --db_password - Password of the database. If not given, a password will be generated ynh_mysql_setup_db () { # Declare an array to define the options of this helper. + local legacy_args=unp declare -Ar args_array=( [u]=db_user= [n]=db_name= [p]=db_password= ) local db_user local db_name @@ -186,6 +194,7 @@ ynh_mysql_setup_db () { # | arg: -n, --db_name - Name of the database ynh_mysql_remove_db () { # Declare an array to define the options of this helper. + local legacy_args=un declare -Ar args_array=( [u]=db_user= [n]=db_name= ) local db_user local db_name @@ -216,6 +225,7 @@ ynh_mysql_remove_db () { # | ret: the corrected name ynh_sanitize_dbid () { # Declare an array to define the options of this helper. + local legacy_args=n declare -Ar args_array=( [n]=db_name= ) local db_name # Manage arguments with getopts diff --git a/data/helpers.d/network b/data/helpers.d/network index 0059702e9..a765d6346 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -1,3 +1,5 @@ +#!/bin/bash + # Normalize the url path syntax # Handle the slash at the beginning of path and its absence at ending # Return a normalized url path @@ -12,6 +14,7 @@ # | arg: -p, --path_url - URL path to normalize before using it ynh_normalize_url_path () { # Declare an array to define the options of this helper. + local legacy_args=p declare -Ar args_array=( [p]=path_url= ) local path_url # Manage arguments with getopts @@ -35,6 +38,7 @@ ynh_normalize_url_path () { # | arg: -p, --port - port to start to search ynh_find_port () { # Declare an array to define the options of this helper. + local legacy_args=p declare -Ar args_array=( [p]=port= ) local port # Manage arguments with getopts @@ -57,6 +61,7 @@ ynh_find_port () { # | arg: -p, --path_url - the web path to check the availability of ynh_webpath_available () { # Declare an array to define the options of this helper. + local legacy_args=dp declare -Ar args_array=( [d]=domain= [p]=path_url= ) local domain local path_url @@ -76,6 +81,7 @@ ynh_webpath_available () { # | arg: -p, --path_url - the web path to be registered ynh_webpath_register () { # Declare an array to define the options of this helper. + local legacy_args=adp declare -Ar args_array=( [a]=app= [d]=domain= [p]=path_url= ) local app local domain diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index fcba2d75a..c4332b60c 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -1,3 +1,5 @@ +#!/bin/bash + n_install_dir="/opt/node_n" node_version_path="$n_install_dir/n/versions/node" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. @@ -61,6 +63,7 @@ ynh_install_nodejs () { # Use n, https://github.com/tj/n to manage the nodejs versions # Declare an array to define the options of this helper. + local legacy_args=n declare -Ar args_array=( [n]=nodejs_version= ) local nodejs_version # Manage arguments with getopts diff --git a/data/helpers.d/package b/data/helpers.d/package index 820a61ef4..485cee957 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -1,3 +1,5 @@ +#!/bin/bash + # Check if apt is free to use, or wait, until timeout. # # [internal] @@ -29,6 +31,7 @@ ynh_wait_dpkg_free() { # | arg: -p, --package - the package name to check ynh_package_is_installed() { # Declare an array to define the options of this helper. + local legacy_args=p declare -Ar args_array=( [p]=package= ) local package # Manage arguments with getopts @@ -48,6 +51,7 @@ ynh_package_is_installed() { # | ret: the version or an empty string ynh_package_version() { # Declare an array to define the options of this helper. + local legacy_args=p declare -Ar args_array=( [p]=package= ) local package # Manage arguments with getopts diff --git a/data/helpers.d/print b/data/helpers.d/print index e605e7e4a..353fa595d 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -1,7 +1,10 @@ +#!/bin/bash + # Print a message to stderr and exit # usage: ynh_die --message=MSG [--ret_code=RETCODE] ynh_die() { # Declare an array to define the options of this helper. + local legacy_args=mc declare -Ar args_array=( [m]=message= [c]=ret_code= ) local message local ret_code @@ -17,6 +20,7 @@ ynh_die() { # usage: ynh_print_info --message="Some message" ynh_print_info() { # Declare an array to define the options of this helper. + local legacy_args=m declare -Ar args_array=( [m]=message= ) local message # Manage arguments with getopts @@ -56,6 +60,7 @@ ynh_print_log () { # | arg: -m, --message - The text to print ynh_print_warn () { # Declare an array to define the options of this helper. + local legacy_args=m declare -Ar args_array=( [m]=message= ) local message # Manage arguments with getopts @@ -70,6 +75,7 @@ ynh_print_warn () { # | arg: -m, --message - The text to print ynh_print_err () { # Declare an array to define the options of this helper. + local legacy_args=m declare -Ar args_array=( [m]=message= ) local message # Manage arguments with getopts diff --git a/data/helpers.d/setting b/data/helpers.d/setting index 29eeecd3c..6f75f6c80 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -1,3 +1,5 @@ +#!/bin/bash + # Get an application setting # # usage: ynh_app_setting_get --app=app --key=key @@ -5,6 +7,7 @@ # | arg: -k, --key - the setting to get ynh_app_setting_get() { # Declare an array to define the options of this helper. + local legacy_args=ak declare -Ar args_array=( [a]=app= [k]=key= ) local app local key @@ -22,6 +25,7 @@ ynh_app_setting_get() { # | arg: -v, --value - the setting value to set ynh_app_setting_set() { # Declare an array to define the options of this helper. + local legacy_args=akv declare -Ar args_array=( [a]=app= [k]=key= [v]=value= ) local app local key @@ -39,6 +43,7 @@ ynh_app_setting_set() { # | arg: -k, --key - the setting to delete ynh_app_setting_delete() { # Declare an array to define the options of this helper. + local legacy_args=ak declare -Ar args_array=( [a]=app= [k]=key= ) local app local key diff --git a/data/helpers.d/string b/data/helpers.d/string index b4dfb596d..739757d43 100644 --- a/data/helpers.d/string +++ b/data/helpers.d/string @@ -1,3 +1,5 @@ +#!/bin/bash + # Generate a random string # # example: pwd=$(ynh_string_random --length=8) @@ -6,6 +8,7 @@ # | arg: -l, --length - the string length to generate (default: 24) ynh_string_random() { # Declare an array to define the options of this helper. + local legacy_args=l declare -Ar args_array=( [l]=length= ) local length # Manage arguments with getopts @@ -29,6 +32,7 @@ ynh_string_random() { # (see sed manual page for more information) ynh_replace_string () { # Declare an array to define the options of this helper. + local legacy_args=mrf declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= ) local match_string local replace_string @@ -55,6 +59,7 @@ ynh_replace_string () { # characters, you can't use some regular expressions and sub-expressions. ynh_replace_special_string () { # Declare an array to define the options of this helper. + local legacy_args=mrf declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= ) local match_string local replace_string diff --git a/data/helpers.d/system b/data/helpers.d/system index 70cc57493..968135f16 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -1,3 +1,5 @@ +#!/bin/bash + # Manage a fail of the script # # [internal] diff --git a/data/helpers.d/user b/data/helpers.d/user index 3947ebe7f..1fa8f3f79 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -1,3 +1,5 @@ +#!/bin/bash + # Check if a YunoHost user exists # # example: ynh_user_exists 'toto' || exit 1 @@ -6,6 +8,7 @@ # | arg: -u, --username - the username to check ynh_user_exists() { # Declare an array to define the options of this helper. + local legacy_args=u declare -Ar args_array=( [u]=username= ) local username # Manage arguments with getopts @@ -24,6 +27,7 @@ ynh_user_exists() { # | ret: string - the key's value ynh_user_get_info() { # Declare an array to define the options of this helper. + local legacy_args=uk declare -Ar args_array=( [u]=username= [k]=key= ) local username local key @@ -50,6 +54,7 @@ ynh_user_list() { # | arg: -u, --username - the username to check ynh_system_user_exists() { # Declare an array to define the options of this helper. + local legacy_args=u declare -Ar args_array=( [u]=username= ) local username # Manage arguments with getopts @@ -65,6 +70,7 @@ ynh_system_user_exists() { # | arg: -h, --home_dir - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home ynh_system_user_create () { # Declare an array to define the options of this helper. + local legacy_args=uh declare -Ar args_array=( [u]=username= [h]=home_dir= ) local username local home_dir @@ -88,6 +94,7 @@ ynh_system_user_create () { # | arg: -u, --username - Name of the system user that will be create ynh_system_user_delete () { # Declare an array to define the options of this helper. + local legacy_args=u declare -Ar args_array=( [u]=username= ) local username # Manage arguments with getopts diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 1488289fa..60ba709ba 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -1,3 +1,5 @@ +#!/bin/bash + # Extract a key from a plain command output # # example: yunohost user info tata --output-as plain | ynh_get_plain_key mail @@ -146,6 +148,7 @@ ynh_backup_before_upgrade () { # | arg: -s, --source_id - Name of the app, if the package contains more than one app ynh_setup_source () { # Declare an array to define the options of this helper. + local legacy_args=ds declare -Ar args_array=( [d]=dest_dir= [s]=source_id= ) local dest_dir local source_id From 7de184a520b9f746f2fe4f8eb944e5438f5bb223 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 4 Jan 2019 17:21:17 +0100 Subject: [PATCH 035/113] Use getopts helpers in mysql, again --- data/helpers.d/mysql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 538b00469..9b4908097 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -40,8 +40,8 @@ ynh_mysql_execute_as_root() { ynh_handle_getopts_args "$@" database="${database:-}" - ynh_mysql_connect_as "root" "$(sudo cat $MYSQL_ROOT_PWD_FILE)" \ - "$database" <<< "$sql" + ynh_mysql_connect_as --user="root" --password="$(sudo cat $MYSQL_ROOT_PWD_FILE)" \ + --database="$database" <<< "$sql" } # Execute a command from a file as root user @@ -59,8 +59,8 @@ ynh_mysql_execute_file_as_root() { ynh_handle_getopts_args "$@" database="${database:-}" - ynh_mysql_connect_as "root" "$(sudo cat $MYSQL_ROOT_PWD_FILE)" \ - "$database" < "$file" + ynh_mysql_connect_as --user="root" --password="$(sudo cat $MYSQL_ROOT_PWD_FILE)" \ + --database="$database" < "$file" } # Create a database and grant optionnaly privilegies to a user @@ -83,7 +83,7 @@ ynh_mysql_create_db() { sql+=" WITH GRANT OPTION;" fi - ynh_mysql_execute_as_root "$sql" + ynh_mysql_execute_as_root --sql="$sql" } # Drop a database @@ -96,7 +96,7 @@ ynh_mysql_create_db() { # usage: ynh_mysql_drop_db db # | arg: db - the database name to drop ynh_mysql_drop_db() { - ynh_mysql_execute_as_root "DROP DATABASE ${1};" + ynh_mysql_execute_as_root --sql="DROP DATABASE ${1};" } # Dump a database @@ -126,7 +126,7 @@ ynh_mysql_dump_db() { # | arg: pwd - the password to identify user by ynh_mysql_create_user() { ynh_mysql_execute_as_root \ - "CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';" + --sql="CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';" } # Check if a mysql user exists @@ -142,7 +142,7 @@ ynh_mysql_user_exists() # Manage arguments with getopts ynh_handle_getopts_args "$@" - if [[ -z $(ynh_mysql_execute_as_root "SELECT User from mysql.user WHERE User = '$user';") ]] + if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]] then return 1 else @@ -157,7 +157,7 @@ ynh_mysql_user_exists() # usage: ynh_mysql_drop_user user # | arg: user - the user name to drop ynh_mysql_drop_user() { - ynh_mysql_execute_as_root "DROP USER '${1}'@'localhost';" + ynh_mysql_execute_as_root --sql="DROP USER '${1}'@'localhost';" } # Create a database, an user and its password. Then store the password in the app's config @@ -210,7 +210,7 @@ ynh_mysql_remove_db () { fi # Remove mysql user if it exists - if $(ynh_mysql_user_exists $db_user); then + if $(ynh_mysql_user_exists --user=$db_user); then ynh_mysql_drop_user $db_user fi } From 1a4e661e4eeadad45faab114ed8a9ff66e8b82fc Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 4 Jan 2019 18:43:56 +0100 Subject: [PATCH 036/113] Keep db_pwd instead of db_password --- data/helpers.d/mysql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 9b4908097..1fa01224a 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -165,26 +165,26 @@ ynh_mysql_drop_user() { # After executing this helper, the password of the created database will be available in $db_pwd # It will also be stored as "mysqlpwd" into the app settings. # -# usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_password=pwd] +# usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] # | arg: -u, --db_user - Owner of the database # | arg: -n, --db_name - Name of the database -# | arg: -p, --db_password - Password of the database. If not given, a password will be generated +# | arg: -p, --db_pwd - Password of the database. If not given, a password will be generated ynh_mysql_setup_db () { # Declare an array to define the options of this helper. local legacy_args=unp - declare -Ar args_array=( [u]=db_user= [n]=db_name= [p]=db_password= ) + declare -Ar args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) local db_user local db_name - local db_password + local db_pwd # Manage arguments with getopts ynh_handle_getopts_args "$@" local new_db_pwd=$(ynh_string_random) # Generate a random password - # If $db_password is not given, use new_db_pwd instead for db_password. - db_password="${db_password:-$new_db_pwd}" + # If $db_pwd is not given, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" - ynh_mysql_create_db "$db_name" "$db_user" "$db_password" # Create the database - ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_password # Store the password in the app's config + ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database + ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd # Store the password in the app's config } # Remove a database if it exists, and the associated user From 50f3291ea7eebb7edc229229f45ce4a5cf66fd1c Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 4 Jan 2019 18:44:40 +0100 Subject: [PATCH 037/113] Fix getopts with blank argument's value There's no reason in that loop where we're looking for values of an option to have a blank cell in the array of arguments. Unless for an option with a missing value. In that case, it's better to ignore this condition and store a blank value in the variable. Otherwise, in case of missing value for an option, with this condition, we enter in an infinite loop, because the shift will stay at 0. --- data/helpers.d/getopts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/getopts b/data/helpers.d/getopts index e9fbe18e8..b3d444767 100644 --- a/data/helpers.d/getopts +++ b/data/helpers.d/getopts @@ -136,7 +136,7 @@ ynh_handle_getopts_args () { for i in `seq 0 $(( ${#all_args[@]} - 1 ))` do # If this argument is an option, end here. - if [ "${all_args[$i]:0:1}" == "-" ] || [ -z "${all_args[$i]}" ] + if [ "${all_args[$i]:0:1}" == "-" ] then # Ignore the first value of the array, which is the option itself if [ "$i" -ne 0 ]; then From 540291a7e072dde610d80e45ec70496d9e2e58e6 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 4 Jan 2019 19:26:54 +0100 Subject: [PATCH 038/113] Ignore useless parameters in legacy mode --- data/helpers.d/getopts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/getopts b/data/helpers.d/getopts index b3d444767..efaa8d065 100644 --- a/data/helpers.d/getopts +++ b/data/helpers.d/getopts @@ -166,8 +166,9 @@ ynh_handle_getopts_args () { # Check if there's getopts arguments if [ "${arguments[0]:0:1}" != "-" ] then - # If not, enter in legacy mode and manage the arguments as positionnal ones. - ynh_print_info --message="! Helper used in legacy mode !" + # If not, enter in legacy mode and manage the arguments as positionnal ones.. + # Dot not echo, to prevent to go through a helper output. But print only in the log. + set -x; echo "! Helper used in legacy mode !" > /dev/null; set +x local i for i in `seq 0 $(( ${#arguments[@]} -1 ))` do @@ -177,6 +178,10 @@ ynh_handle_getopts_args () { getopts_parameters=${legacy_args:-${getopts_parameters//:}} # Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument. option_flag=${getopts_parameters:$i:1} + if [ -z "$option_flag" ]; then + ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored." + continue + fi # Use the long option, corresponding to the option_flag, as a variable # (e.g. for [u]=user, 'user' will be used as a variable) # Also, remove '=' at the end of the long option From bf443f2ec0fc270bff4730200de2921152d8b04b Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 4 Jan 2019 19:30:07 +0100 Subject: [PATCH 039/113] db_pwd shouldn't be local --- data/helpers.d/mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 1fa01224a..fa1a61dab 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -175,7 +175,7 @@ ynh_mysql_setup_db () { declare -Ar args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) local db_user local db_name - local db_pwd + db_pwd="" # Manage arguments with getopts ynh_handle_getopts_args "$@" From 0a3e5c60ead612c921f3a31aadfa71e32359ca7a Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 5 Jan 2019 15:09:17 +0100 Subject: [PATCH 040/113] source getopts in filesystem Many core backup script source /usr/share/yunohost/helpers.d/filesystem only. Not sure it's a wonderful idea... --- data/helpers.d/filesystem | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 2e9f93166..0f2e06b64 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -1,5 +1,7 @@ #!/bin/bash +source /usr/share/yunohost/helpers.d/getopts + CAN_BIND=${CAN_BIND:-1} # Add a file or a directory to the list of paths to backup From 9fb302fe043a368d27f18ca05cd843841a2ef5f2 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 13 Jan 2019 14:24:17 +0100 Subject: [PATCH 041/113] [fix] Replace manifest_path by manifest manifest_path isn't defined --- data/helpers.d/system | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index f09343953..a93b3ea6f 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -85,10 +85,10 @@ ynh_app_upstream_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest_path" ]; then - manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + if [ ! -e "$manifest" ]; then + manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place fi - version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/~ynh*/}" } @@ -107,10 +107,10 @@ ynh_app_package_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest_path" ]; then - manifest_path="../settings/manifest.json" # Into the restore script, the manifest is not at the same place + if [ ! -e "$manifest" ]; then + manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place fi - version_key=$(ynh_read_manifest --manifest="$manifest_path" --manifest_key="version") + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/*~ynh/}" } From 08869c329c5ac58645544f5672d53ab61927fa7b Mon Sep 17 00:00:00 2001 From: frju365 Date: Mon, 21 Jan 2019 21:17:33 +0100 Subject: [PATCH 042/113] Update yunohost_admin.conf --- data/templates/nginx/plain/yunohost_admin.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 3de66e3e6..06d1ef09c 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -68,7 +68,7 @@ server { if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { return 403; } - + add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; } From a42df2d8fe3371b5e1d5c0a7a325fa8524e7fe28 Mon Sep 17 00:00:00 2001 From: frju365 Date: Mon, 21 Jan 2019 21:39:07 +0100 Subject: [PATCH 043/113] Update yunohost_admin.conf --- data/templates/nginx/plain/yunohost_admin.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 06d1ef09c..917ad0d5e 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -65,9 +65,6 @@ server { location /yunohost { # Block crawlers bot - if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { - return 403; - } add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; From 7b7030544876178e606c389606766eff89da21c4 Mon Sep 17 00:00:00 2001 From: frju365 Date: Sat, 26 Jan 2019 23:51:22 +0100 Subject: [PATCH 044/113] Update yunohost_admin.conf --- data/templates/nginx/plain/yunohost_admin.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 917ad0d5e..24ea25072 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -65,6 +65,10 @@ server { location /yunohost { # Block crawlers bot + if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { + return 403; + } + # X-Robots-Tag to precise the rules applied. add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; From 63d62b4f7a77ab780d43a320b5d4c1f94b36d459 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 28 Jan 2019 00:42:56 +0100 Subject: [PATCH 045/113] [enh] Allow to dig deeper into an archive with ynh_setup_source --- data/helpers.d/utils | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 43dabfcd4..699440edb 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -118,6 +118,8 @@ ynh_backup_before_upgrade () { # SOURCE_FORMAT=tar.gz # # (Optional) Put false if sources are directly in the archive root # # default: true +# # Instead of true, SOURCE_IN_SUBDIR could be the number of sub directories +# # to remove. # SOURCE_IN_SUBDIR=false # # (Optionnal) Name of the local archive (offline setup support) # # default: ${src_id}.${src_format} @@ -136,6 +138,8 @@ ynh_backup_before_upgrade () { # If it's ok, the source archive will be uncompressed in $dest_dir. If the # SOURCE_IN_SUBDIR is true, the first level directory of the archive will be # removed. +# If SOURCE_IN_SUBDIR is a numeric value, 2 for example, the 2 first level +# directories will be removed # # Finally, patches named sources/patches/${src_id}-*.patch and extra files in # sources/extra_files/$src_id will be applied to dest_dir @@ -182,7 +186,7 @@ ynh_setup_source () { # Extract source into the app dir mkdir -p "$dest_dir" - + if ! "$src_extract" then mv $src_filename $dest_dir @@ -200,8 +204,14 @@ ynh_setup_source () { fi else local strip="" - if $src_in_subdir ; then - strip="--strip-components 1" + if [ "$src_in_subdir" != "false" ] + then + if [ "$src_in_subdir" == "true" ]; then + local sub_dirs=1 + else + local sub_dirs="$src_in_subdir" + fi + strip="--strip-components $sub_dirs" fi if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]] ; then tar -xf $src_filename -C "$dest_dir" $strip From b9fa901bda11a97c9460ac5f426bc38a0e21c6c3 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 28 Jan 2019 16:31:54 +0100 Subject: [PATCH 046/113] Add ynh_script_progression --- data/helpers.d/print | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/data/helpers.d/print b/data/helpers.d/print index 2f451bc24..1d7f6f6fa 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -124,3 +124,81 @@ ynh_print_ON () { # Print an echo only for the log, to be able to know that ynh_print_ON has been called. echo ynh_print_ON > /dev/null } + +# Print a message as INFO and show progression during an app script +# +# usage: ynh_script_progression --message=message [--weight=weight] [--time] +# | arg: -m, --message= - The text to print +# | arg: -w, --weight= - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script. +# | arg: -t, --time= - Print the execution time since the last call to this helper. Especially usefull to define weights. +# | arg: -l, --last= - Use for the last call of the helper, to fill te progression bar. +increment_progression=0 +previous_weight=0 +# Define base_time when the file is sourced +base_time=$(date +%s) +ynh_script_progression () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=message= [w]=weight= [t]=time [l]=last ) + local message + local weight + local time + local last + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + weight=${weight:-1} + time=${time:-0} + last=${last:-0} + + # Get execution time since the last $base_time + local exec_time=$(( $(date +%s) - $base_time )) + base_time=$(date +%s) + + # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented. + local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)" + # Get the number of call with a weight value + local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0) + + # Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight + local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]].*\)/\1/g')" + # Get the weight of each occurrences of 'ynh_script_progression' in the script using -w + local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]].*\)/\1/g')" + # Each value will be on a different line. + # Remove each 'end of line' and replace it by a '+' to sum the values. + local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 )) + + # max_progression is a total number of calls to this helper. + # Less the number of calls with a weight value. + # Plus the total of weight values + local max_progression=$(( $helper_calls - $weight_calls + $weight_values )) + + # Increment each execution of ynh_script_progression in this script by the weight of the previous call. + increment_progression=$(( $increment_progression + $previous_weight )) + # Store the weight of the current call in $previous_weight for next call + previous_weight=$weight + + # Set the scale of the progression bar + local scale=20 + # progress_string(1,2) should have the size of the scale. + local progress_string1="####################" + local progress_string0="...................." + + # Reduce $increment_progression to the size of the scale + if [ $last -eq 0 ] + then + local effective_progression=$(( $increment_progression * $scale / $max_progression )) + # If last is specified, fill immediately the progression_bar + else + local effective_progression=$scale + fi + + # Build $progression_bar from progress_string(1,2) according to $effective_progression + local progression_bar="${progress_string1:0:$effective_progression}${progress_string0:0:$(( $scale - $effective_progression ))}" + + local print_exec_time="" + if [ $time -eq 1 ] + then + print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]" + fi + + ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" +} From 68b01ae6ec529f04e35ae02daf518a540ab6998a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 29 Jan 2019 01:51:52 +0100 Subject: [PATCH 047/113] Warn the user that ynh_secure_remove should be used with only one argument --- data/helpers.d/filesystem | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index dfea026b6..e8f5d349e 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -348,6 +348,11 @@ ynh_secure_remove () { /var/www \ /home/yunohost.app" + if [[ -n "$2" ]] + then + echo "/!\ Packager ! You provided a second argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." >&2 + fi + if [[ "$forbidden_path" =~ "$path_to_remove" \ # Match all paths or subpaths in $forbidden_path || "$path_to_remove" =~ ^/[[:alnum:]]+$ \ From a26994fd70d9f8b58027548b9579e9818ed085af Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 29 Jan 2019 23:04:14 +0100 Subject: [PATCH 048/113] Update DNS resolver list according to diyisp.org --- data/templates/dnsmasq/plain/resolv.dnsmasq.conf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index bc36ef365..f2bb846f7 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -14,15 +14,17 @@ nameserver 80.67.169.40 nameserver 80.67.188.188 # (FR) ARN nameserver 89.234.141.66 +# (FR) Aquilenet +nameserver 185.233.100.100 +nameserver 185.233.100.101 # (FR) gozmail / grifon nameserver 89.234.186.18 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 -# (FR) Aquilenet [added manually, following comments from @sachaz] -nameserver 141.255.128.100 -nameserver 141.255.128.101 -# (DE) CCC Berlin -nameserver 213.73.91.35 +# (DE) CCC Berlin (not working since Dec 2018 ?) +#nameserver 213.73.91.35 +# (DE) AS250 +nameserver 194.150.168.168 # (DE) Ideal-Hosting nameserver 84.200.69.80 nameserver 84.200.70.40 From d5bf9a61b87b2b1b28c5881d76ebbb7fd9cecfd6 Mon Sep 17 00:00:00 2001 From: opi Date: Wed, 30 Jan 2019 14:09:59 +0100 Subject: [PATCH 049/113] [fix] Remove old SMTP port (465) from Fail2ban jail.conf. Fix #1283 --- data/templates/fail2ban/jail.conf | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/data/templates/fail2ban/jail.conf b/data/templates/fail2ban/jail.conf index 05eb7e7a8..9b4d39f17 100644 --- a/data/templates/fail2ban/jail.conf +++ b/data/templates/fail2ban/jail.conf @@ -513,27 +513,27 @@ logpath = %(vsftpd_log)s # ASSP SMTP Proxy Jail [assp] -port = smtp,465,submission +port = smtp,submission logpath = /root/path/to/assp/logs/maillog.txt [courier-smtp] -port = smtp,465,submission +port = smtp,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix] -port = smtp,465,submission +port = smtp,submission logpath = %(postfix_log)s backend = %(postfix_backend)s [postfix-rbl] -port = smtp,465,submission +port = smtp,submission logpath = %(postfix_log)s backend = %(postfix_backend)s maxretry = 1 @@ -541,14 +541,14 @@ maxretry = 1 [sendmail-auth] -port = submission,465,smtp +port = submission,smtp logpath = %(syslog_mail)s backend = %(syslog_backend)s [sendmail-reject] -port = smtp,465,submission +port = smtp,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s @@ -556,7 +556,7 @@ backend = %(syslog_backend)s [qmail-rbl] filter = qmail -port = smtp,465,submission +port = smtp,submission logpath = /service/qmail/log/main/current @@ -564,14 +564,14 @@ logpath = /service/qmail/log/main/current # but can be set by syslog_facility in the dovecot configuration. [dovecot] -port = pop3,pop3s,imap,imaps,submission,465,sieve +port = pop3,pop3s,imap,imaps,submission,sieve logpath = %(dovecot_log)s backend = %(dovecot_backend)s [sieve] -port = smtp,465,submission +port = smtp,submission logpath = %(dovecot_log)s backend = %(dovecot_backend)s @@ -584,19 +584,19 @@ logpath = %(solidpop3d_log)s [exim] -port = smtp,465,submission +port = smtp,submission logpath = %(exim_main_log)s [exim-spam] -port = smtp,465,submission +port = smtp,submission logpath = %(exim_main_log)s [kerio] -port = imap,smtp,imaps,465 +port = imap,smtp,imaps logpath = /opt/kerio/mailserver/store/logs/security.log @@ -607,14 +607,14 @@ logpath = /opt/kerio/mailserver/store/logs/security.log [courier-auth] -port = smtp,465,submission,imaps,pop3,pop3s +port = smtp,submission,imaps,pop3,pop3s logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix-sasl] -port = smtp,465,submission,imap,imaps,pop3,pop3s +port = smtp,submission,imap,imaps,pop3,pop3s # You might consider monitoring /var/log/mail.warn instead if you are # running postfix since it would provide the same log lines at the # "warn" level but overall at the smaller filesize. @@ -631,7 +631,7 @@ backend = %(syslog_backend)s [squirrelmail] -port = smtp,465,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks +port = smtp,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log From 40382acbfbf9677eac675d9457109a03368e3ef6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 30 Jan 2019 15:22:32 +0100 Subject: [PATCH 050/113] Update gozmail IP --- data/templates/dnsmasq/plain/resolv.dnsmasq.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index f2bb846f7..c7cd29d2c 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -18,7 +18,7 @@ nameserver 89.234.141.66 nameserver 185.233.100.100 nameserver 185.233.100.101 # (FR) gozmail / grifon -nameserver 89.234.186.18 +nameserver 80.67.190.200 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 # (DE) CCC Berlin (not working since Dec 2018 ?) From 4e7a4f68273087184f311f19d9dcfc3fbe153add Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 30 Jan 2019 15:25:02 +0100 Subject: [PATCH 051/113] Update IP for CCC Berlin --- data/templates/dnsmasq/plain/resolv.dnsmasq.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index c7cd29d2c..7eed1142f 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -21,8 +21,8 @@ nameserver 185.233.100.101 nameserver 80.67.190.200 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 -# (DE) CCC Berlin (not working since Dec 2018 ?) -#nameserver 213.73.91.35 +# (DE) CCC Berlin +nameserver 195.160.173.53 # (DE) AS250 nameserver 194.150.168.168 # (DE) Ideal-Hosting From d77bc92b6e4be77e30b2600ea099c1805f4a39fb Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Thu, 31 Jan 2019 17:54:25 +0100 Subject: [PATCH 052/113] Check if dpkg is broken --- data/helpers.d/package | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/data/helpers.d/package b/data/helpers.d/package index 2cbca4840..6bd655fe0 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -15,6 +15,21 @@ ynh_wait_dpkg_free() { # Sleep an exponential time at each round sleep $(( try * try )) else + # Check if dpkg hasn't been interrupted and is fully available. + # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + local dpkg_dir="/var/lib/dpkg/updates/" + + # For each file in $dpkg_dir + while read dpkg_file <&9 + do + # Check if the name of this file contains only numbers. + if echo "$dpkg_file" | grep -Pq "^[[:digit:]]*$" + then + # If so, that a remaining of dpkg. + ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + return 1 + fi + done 9<<< "$(ls -1 $dpkg_dir)" return 0 fi done From db83dc089334ccf0314da5d9c06cb82483d90545 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 3 Feb 2019 03:20:19 +0100 Subject: [PATCH 053/113] [wip] Fix password in log --- bin/yunohost-api | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/yunohost-api b/bin/yunohost-api index 93d44c256..6e47df7ef 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -149,6 +149,11 @@ def _init_moulinette(use_websocket=True, debug=False, verbose=False): 'handlers': [], 'propagate': True, }, + 'gnupg': { + 'level': 'INFO', + 'handlers': [], + 'propagate': False, + }, }, 'root': { 'level': level, From 0267a798a75976ccfbc175c32d0f58905bc89034 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 6 Feb 2019 19:27:43 +0100 Subject: [PATCH 054/113] [fix] sterror doesn't exist most of the time --- src/yunohost/app.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 6f9402f83..0f73aea43 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1173,7 +1173,7 @@ def app_makedefault(operation_logger, auth, app, domain=None): with open('/etc/ssowat/conf.json.persistent') as json_conf: ssowat_conf = json.loads(str(json_conf.read())) except ValueError as e: - raise YunohostError('ssowat_persistent_conf_read_error', error=e.strerror) + raise YunohostError('ssowat_persistent_conf_read_error', error=e) except IOError: ssowat_conf = {} @@ -1186,7 +1186,7 @@ def app_makedefault(operation_logger, auth, app, domain=None): with open('/etc/ssowat/conf.json.persistent', 'w+') as f: json.dump(ssowat_conf, f, sort_keys=True, indent=4) except IOError as e: - raise YunohostError('ssowat_persistent_conf_write_error', error=e.strerror) + raise YunohostError('ssowat_persistent_conf_write_error', error=e) os.system('chmod 644 /etc/ssowat/conf.json.persistent') @@ -1824,7 +1824,7 @@ def _extract_app_from_file(path, remove=False): except IOError: raise YunohostError('app_install_files_invalid') except ValueError as e: - raise YunohostError('app_manifest_invalid', error=e.strerror) + raise YunohostError('app_manifest_invalid', error=e) logger.debug(m18n.n('done')) @@ -1917,7 +1917,7 @@ def _fetch_app_from_git(app): except subprocess.CalledProcessError: raise YunohostError('app_sources_fetch_failed') except ValueError as e: - raise YunohostError('app_manifest_invalid', error=e.strerror) + raise YunohostError('app_manifest_invalid', error=e) else: logger.debug(m18n.n('done')) @@ -1970,7 +1970,7 @@ def _fetch_app_from_git(app): except subprocess.CalledProcessError: raise YunohostError('app_sources_fetch_failed') except ValueError as e: - raise YunohostError('app_manifest_invalid', error=e.strerror) + raise YunohostError('app_manifest_invalid', error=e) else: logger.debug(m18n.n('done')) @@ -2250,7 +2250,7 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): try: user_info(auth, arg_value) except YunohostError as e: - raise YunohostError('app_argument_invalid', name=arg_name, error=e.strerror) + raise YunohostError('app_argument_invalid', name=arg_name, error=e) elif arg_type == 'app': if not _is_installed(arg_value): raise YunohostError('app_argument_invalid', name=arg_name, error=m18n.n('app_unknown')) From 169ae96f0a52129e505972061709af0e2a8ec0ad Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 6 Feb 2019 19:32:35 +0100 Subject: [PATCH 055/113] [mod] please flake8 by respecting pep8 and removing except: --- src/yunohost/app.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0f73aea43..0bca68787 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -734,8 +734,8 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on if answer.upper() != "Y": raise YunohostError("aborting") - raw_app_list = app_list(raw=True) + if app in raw_app_list or ('@' in app) or ('http://' in app) or ('https://' in app): if app in raw_app_list: state = raw_app_list[app].get("state", "notworking") @@ -1209,8 +1209,8 @@ def app_setting(app, key, value=None, delete=False): if value is None and not delete: try: return app_settings[key] - except: - logger.debug("cannot get app setting '%s' for '%s'", key, app) + except Exception as e: + logger.debug("cannot get app setting '%s' for '%s' (%s)", key, app, e) return None else: if delete and key in app_settings: @@ -1394,7 +1394,8 @@ def app_ssowatconf(auth): try: apps_list = app_list(installed=True)['apps'] - except: + except Exception as e: + logger.debug("cannot get installed app list because %s", e) apps_list = [] def _get_setting(settings, name): @@ -1907,7 +1908,7 @@ def _fetch_app_from_git(app): # we will be able to use it. Without this option all the history # of the submodules repo is downloaded. subprocess.check_call([ - 'git', 'clone', '-b', branch, '--single-branch', '--recursive', '--depth=1', url, + 'git', 'clone', '-b', branch, '--single-branch', '--recursive', '--depth=1', url, extracted_app_folder]) subprocess.check_call([ 'git', 'reset', '--hard', branch @@ -1925,8 +1926,8 @@ def _fetch_app_from_git(app): manifest['remote'] = {'type': 'git', 'url': url, 'branch': branch} try: revision = _get_git_last_commit_hash(url, branch) - except: - pass + except Exception as e: + logger.debug("cannot get last commit hash because: %s ", e) else: manifest['remote']['revision'] = revision else: From 060370ef889829d6bacf08f62bd7a7b5ad53177d Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 6 Feb 2019 22:46:21 +0100 Subject: [PATCH 056/113] Fix #635 on ynh_secure_remove --- data/helpers.d/filesystem | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 6e0edfa37..88d51750b 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -392,10 +392,10 @@ ynh_secure_remove () { /var/www \ /home/yunohost.app" - if [[ -n "$2" ]] - then - echo "/!\ Packager ! You provided a second argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." >&2 - fi + if [ $# -ge 2 ] + then + echo "/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." >&2 + fi if [[ "$forbidden_path" =~ "$file" \ # Match all paths or subpaths in $forbidden_path From f0e65750623e0a6646441ba212ffc12c5388b3e3 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 3 Feb 2019 03:20:19 +0100 Subject: [PATCH 057/113] [wip] Fix password in log --- bin/yunohost-api | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/yunohost-api b/bin/yunohost-api index 93d44c256..6e47df7ef 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -149,6 +149,11 @@ def _init_moulinette(use_websocket=True, debug=False, verbose=False): 'handlers': [], 'propagate': True, }, + 'gnupg': { + 'level': 'INFO', + 'handlers': [], + 'propagate': False, + }, }, 'root': { 'level': level, From 0b98df85f1f75766bbf72cf9cbc266c50f9536a5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 29 Jan 2019 23:04:14 +0100 Subject: [PATCH 058/113] Update DNS resolver list according to diyisp.org --- data/templates/dnsmasq/plain/resolv.dnsmasq.conf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index bc36ef365..f2bb846f7 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -14,15 +14,17 @@ nameserver 80.67.169.40 nameserver 80.67.188.188 # (FR) ARN nameserver 89.234.141.66 +# (FR) Aquilenet +nameserver 185.233.100.100 +nameserver 185.233.100.101 # (FR) gozmail / grifon nameserver 89.234.186.18 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 -# (FR) Aquilenet [added manually, following comments from @sachaz] -nameserver 141.255.128.100 -nameserver 141.255.128.101 -# (DE) CCC Berlin -nameserver 213.73.91.35 +# (DE) CCC Berlin (not working since Dec 2018 ?) +#nameserver 213.73.91.35 +# (DE) AS250 +nameserver 194.150.168.168 # (DE) Ideal-Hosting nameserver 84.200.69.80 nameserver 84.200.70.40 From f050db0fbf7efcaf85b9e5b1d6c7b1f834505398 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 30 Jan 2019 15:22:32 +0100 Subject: [PATCH 059/113] Update gozmail IP --- data/templates/dnsmasq/plain/resolv.dnsmasq.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index f2bb846f7..c7cd29d2c 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -18,7 +18,7 @@ nameserver 89.234.141.66 nameserver 185.233.100.100 nameserver 185.233.100.101 # (FR) gozmail / grifon -nameserver 89.234.186.18 +nameserver 80.67.190.200 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 # (DE) CCC Berlin (not working since Dec 2018 ?) From ed481e05d832022e13074f7478d03477685d84c6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 30 Jan 2019 15:25:02 +0100 Subject: [PATCH 060/113] Update IP for CCC Berlin --- data/templates/dnsmasq/plain/resolv.dnsmasq.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index c7cd29d2c..7eed1142f 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -21,8 +21,8 @@ nameserver 185.233.100.101 nameserver 80.67.190.200 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 -# (DE) CCC Berlin (not working since Dec 2018 ?) -#nameserver 213.73.91.35 +# (DE) CCC Berlin +nameserver 195.160.173.53 # (DE) AS250 nameserver 194.150.168.168 # (DE) Ideal-Hosting From 45d2001692256c9eec59f3b65804d624ba985f0e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 Feb 2019 15:20:50 +0100 Subject: [PATCH 061/113] Update changelog for 3.4.2.3 --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 6f930aed9..3f3a6a5ef 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +yunohost (3.4.2.3) stable; urgency=low + + - [fix] Admin password appearing in logs after logging in on webadmin + - [fix] Update friendly DNS resolver list + + -- Alexandre Aubin Thu, 07 Feb 2019 03:20:10 +0000 + yunohost (3.4.2.2) stable; urgency=low - Silly bug in migraton 8 :| From a742ca8939d1d136d1b998c96d7e4fb4046dcc3c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 9 Feb 2019 15:40:29 +0100 Subject: [PATCH 062/113] Attempt to get rid of annoying 'unable to initialize frontend' messages --- data/helpers.d/package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/package b/data/helpers.d/package index c0617abb2..1b7c38c5c 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -150,7 +150,7 @@ ynh_package_install_from_equivs () { cp "$controlfile" "${TMPDIR}/control" (cd "$TMPDIR" equivs-build ./control 1> /dev/null - dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) + DEBIAN_FRONTEND=noninteractive dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) ynh_package_install -f || ynh_die --message="Unable to install dependencies" [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR # Remove the temp dir. From 6ba12bb9ae77c60653969b6bc999d6be739f7685 Mon Sep 17 00:00:00 2001 From: Kayou Date: Sat, 9 Feb 2019 23:04:21 +0100 Subject: [PATCH 063/113] Fix ynh_local_curl --- data/helpers.d/utils | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 360174847..94ba8bd97 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -257,7 +257,15 @@ ynh_setup_source () { # | arg: ... - (Optionnal) More POST keys and values ynh_local_curl () { # Define url of page to curl - local full_page_url=https://localhost$path_url$1 + path_url=$(ynh_normalize_url_path $path_url) + local_page=$(ynh_normalize_url_path $1) + full_path=$path_url$local_page + + if [ "${path_url}" == "/" ]; then + full_path=$local_page + fi + + local full_page_url=https://localhost$full_path # Concatenate all other arguments with '&' to prepare POST data local POST_data="" From a0dfae269f2d1c6f570d55cdeef2d8a495a73a59 Mon Sep 17 00:00:00 2001 From: Kayou Date: Sat, 9 Feb 2019 23:13:48 +0100 Subject: [PATCH 064/113] Update utils --- data/helpers.d/utils | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 94ba8bd97..40bef7b88 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -258,8 +258,8 @@ ynh_setup_source () { ynh_local_curl () { # Define url of page to curl path_url=$(ynh_normalize_url_path $path_url) - local_page=$(ynh_normalize_url_path $1) - full_path=$path_url$local_page + local local_page=$(ynh_normalize_url_path $1) + local full_path=$path_url$local_page if [ "${path_url}" == "/" ]; then full_path=$local_page From 76e1c2e3955f599d51174a8bf8a68d64a301f8bf Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 10 Feb 2019 20:51:40 +0100 Subject: [PATCH 065/113] Fix template variable in ynh_add_systemd_config Merged as a micro decision, as it's only a small mistake. --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 5e6d564cf..c96315238 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -121,7 +121,7 @@ ynh_add_systemd_config () { # Manage arguments with getopts ynh_handle_getopts_args "$@" local service="${service:-$app}" - local template="${nonappend:-systemd.service}" + local template="${template:-systemd.service}" finalsystemdconf="/etc/systemd/system/$service.service" ynh_backup_if_checksum_is_different --file="$finalsystemdconf" From 573fab426834c575596e93c0f4ea7403fd47a4f4 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 10 Feb 2019 21:23:56 +0100 Subject: [PATCH 066/113] [fix] Escape double quote before eval in getopts Work in progress... Need to be tested. --- data/helpers.d/getopts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data/helpers.d/getopts b/data/helpers.d/getopts index efaa8d065..7055325f1 100644 --- a/data/helpers.d/getopts +++ b/data/helpers.d/getopts @@ -150,6 +150,9 @@ ynh_handle_getopts_args () { # If there's already another value for this option, add a ; before adding the new value eval ${option_var}+="\;" fi + # Escape double quote to prevent any interpretation during the eval + all_args[$i]="${all_args[$i]//\"/\\\"}" + eval ${option_var}+=\"${all_args[$i]}\" shift_value=$(( shift_value + 1 )) fi @@ -188,6 +191,9 @@ ynh_handle_getopts_args () { # The variable name will be stored in 'option_var' local option_var="${args_array[$option_flag]%=}" + # Escape double quote to prevent any interpretation during the eval + arguments[$i]="${arguments[$i]//\"/\\\"}" + # Store each value given as argument in the corresponding variable # The values will be stored in the same order than $args_array eval ${option_var}+=\"${arguments[$i]}\" From 3dbe9af7ddbaa082b22f385de13529384a6e0c19 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 10 Feb 2019 23:43:27 +0100 Subject: [PATCH 067/113] Create debug --- data/helpers.d/debug | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 data/helpers.d/debug diff --git a/data/helpers.d/debug b/data/helpers.d/debug new file mode 100644 index 000000000..a8b7c8d69 --- /dev/null +++ b/data/helpers.d/debug @@ -0,0 +1,59 @@ +#!/bin/bash + +# Debugger for app packagers +# +# usage: ynh_debug [--message=message] [--trace=1/0] +# | arg: -m, --message= - The text to print +# | arg: -t, --trace= - Turn on or off the trace of the script. Usefull to trace nonly a small part of a script. +ynh_debug () { + # Disable set xtrace for the helper itself, to not pollute the debug log + set +x + # Declare an array to define the options of this helper. + local legacy_args=mt + declare -Ar args_array=( [m]=message= [t]=trace= ) + local message + local trace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Redisable xtrace, ynh_handle_getopts_args set it back + set +x + message=${message:-} + trace=${trace:-} + + if [ -n "$message" ] + then + ynh_print_log "\e[34m\e[1m[DEBUG]\e[0m ${message}" >&2 + fi + + if [ "$trace" == "1" ] + then + ynh_debug --message="Enable debugging" + set +x + # Get the current file descriptor of xtrace + old_bash_xtracefd=$BASH_XTRACEFD + # Add the current file name and the line number of any command currently running while tracing. + PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: ' + # Force xtrace to stderr + BASH_XTRACEFD=2 + fi + if [ "$trace" == "0" ] + then + ynh_debug --message="Disable debugging" + set +x + # Put xtrace back to its original fild descriptor + BASH_XTRACEFD=$old_bash_xtracefd + fi + # Renable set xtrace + set -x +} + +# Execute a command and print the result as debug +# +# usage: ynh_debug_exec command to execute +# usage: ynh_debug_exec "command to execute | following command" +# In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# +# | arg: command - command to execute +ynh_debug_exec () { + ynh_debug --message="$(eval $@)" +} From b6c5b1fce933847a3a29e7e26c747792271b13b6 Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 11 Feb 2019 16:45:40 +0100 Subject: [PATCH 068/113] Don't normilize path_url --- data/helpers.d/utils | 1 - 1 file changed, 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 40bef7b88..5ba2946a2 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -257,7 +257,6 @@ ynh_setup_source () { # | arg: ... - (Optionnal) More POST keys and values ynh_local_curl () { # Define url of page to curl - path_url=$(ynh_normalize_url_path $path_url) local local_page=$(ynh_normalize_url_path $1) local full_path=$path_url$local_page From 2065b8914262d09bcaee92fb1c9fba7b4ab87ffa Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 12 Feb 2019 00:28:39 +0100 Subject: [PATCH 069/113] Use jq instead of python3 --- data/helpers.d/system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index a93b3ea6f..39af0092f 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -67,7 +67,7 @@ ynh_read_manifest () { # Manage arguments with getopts ynh_handle_getopts_args "$@" - python3 -c "import sys, json;print(json.load(open('$manifest', encoding='utf-8'))['$manifest_key'])" + jq ".$manifest_key" "$manifest" --raw-output } # Read the upstream version from the manifest From 2f1a517a428849c0a252735bee1d5f59c8b13814 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 12 Feb 2019 20:17:15 +0100 Subject: [PATCH 070/113] Print a diff of the files when backup by ynh_backup_if_checksum_is_different --- data/helpers.d/filesystem | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 88d51750b..903f4b6a0 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -352,7 +352,8 @@ ynh_backup_if_checksum_is_different () { local backup_file="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" sudo mkdir -p "$(dirname "$backup_file")" sudo cp -a "$file" "$backup_file" # Backup the current file - echo "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file" >&2 + ynh_print_info "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file" + ynh_print_info "$(git diff --no-index --patch-with-stat $backup_file $file)" echo "$backup_file" # Return the name of the backup file fi fi From 73986ab10f2f44ceaff3c8f33460368461017066 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 12 Feb 2019 21:47:40 +0100 Subject: [PATCH 071/113] Be clever... diff the file after modification... --- data/helpers.d/filesystem | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 903f4b6a0..3f2b69ed2 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -323,6 +323,12 @@ ynh_store_file_checksum () { local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1) + + if [ -n "${backup_file_checksum-}" ] + then + ynh_print_info "$(diff --report-identical-files --unified --color=always $backup_file_checksum $file)" + fi + unset backup_file_checksum } # Verify the checksum and backup the file if it's different @@ -345,16 +351,16 @@ ynh_backup_if_checksum_is_different () { local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + backup_file_checksum="" if [ -n "$checksum_value" ] then # Proceed only if a value was stored into the app settings if ! echo "$checksum_value $file" | sudo md5sum -c --status then # If the checksum is now different - local backup_file="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" - sudo mkdir -p "$(dirname "$backup_file")" - sudo cp -a "$file" "$backup_file" # Backup the current file - ynh_print_info "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file" - ynh_print_info "$(git diff --no-index --patch-with-stat $backup_file $file)" - echo "$backup_file" # Return the name of the backup file + backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" + sudo mkdir -p "$(dirname "$backup_file_checksum")" + sudo cp -a "$file" "$backup_file_checksum" # Backup the current file + ynh_print_info "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + echo "$backup_file_checksum" # Return the name of the backup file fi fi } From 6c925c04c334b3b17ebde3e5e461648e4e20d5a5 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 12 Feb 2019 22:08:37 +0100 Subject: [PATCH 072/113] Fail with ynh_print_info --- data/helpers.d/filesystem | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 3f2b69ed2..713c3890b 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -324,10 +324,14 @@ ynh_store_file_checksum () { local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1) + # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup if [ -n "${backup_file_checksum-}" ] then - ynh_print_info "$(diff --report-identical-files --unified --color=always $backup_file_checksum $file)" + # Print the diff between the previous file and the new one. + # diff return 1 if the files are different, so the || true + diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true fi + # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. unset backup_file_checksum } @@ -351,6 +355,7 @@ ynh_backup_if_checksum_is_different () { local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum backup_file_checksum="" if [ -n "$checksum_value" ] then # Proceed only if a value was stored into the app settings From 140ae8e51a74e9db693b954ea22e08038ff1f5d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Tue, 12 Feb 2019 21:13:49 +0100 Subject: [PATCH 073/113] Change the way to manage the jail and filter file and improve documentations Using a template file make more easy to use a custom failregex. It also give the possiblitity to use custom settings in the fail2ban config --- data/helpers.d/backend | 116 ++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 47 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 6b5ca1d37..ac4d4ed94 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -186,59 +186,85 @@ ynh_remove_fpm_config () { # Create a dedicated fail2ban config (jail and filter conf files) # -# usage: ynh_add_fail2ban_config log_file filter [max_retry [ports]] -# | arg: -l, --logpath= - Log file to be checked by fail2ban -# | arg: -r, --failregex= - Failregex to be looked for by fail2ban -# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# usage: ynh_add_fail2ban_config "list of others variables to replace" +# +# | arg: list of others variables to replace separeted by a space +# | for example : 'var_1 var_2 ...' +# +# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf +# __APP__ by $app +# +# You can dynamically replace others variables by example : +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# +# Note about the "failregex" option: +# regex to match the password failure messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# Note that the logfile need to exist before to call this helper !! +# +# Generally your template will look like that by example (for synapse): +# +# f2b_jail.conf: +# [__APP__] +# enabled = true +# port = http,https +# filter = __APP__ +# logpath = /var/log/__APP__/logfile.log +# maxretry = 3 +# +# f2b_filter.conf: +# [INCLUDES] +# before = common.conf +# [Definition] +# +# # Part of regex definition (just used to make more easy to make the global regex) +# __synapse_start_line = .? \- synapse\..+ \- +# +# # Regex definition. +# failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ +# +# ignoreregex = +# +# To validate your regex you can test with this command: +# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf ynh_add_fail2ban_config () { - # Declare an array to define the options of this helper. - declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= ) - local logpath - local failregex - local max_retry - local ports - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - max_retry=${max_retry:-3} - ports=${ports:-http,https} - - test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + local others_var=${1:-} finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" - ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1 - ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1 + ynh_backup_if_checksum_is_different "$finalfail2banjailconf" + ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" - tee $finalfail2banjailconf <&2 echo "WARNING${fail2ban_error#*WARNING}" >&2 fi @@ -250,9 +276,5 @@ EOF ynh_remove_fail2ban_config () { ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" - if [ "$(lsb_release --codename --short)" != "jessie" ]; then - systemctl reload fail2ban - else - systemctl restart fail2ban - fi + systemctl try-reload-or-restart fail2ban } From e1ccab212a7241d1e76b955b6699c20c10bfc7c0 Mon Sep 17 00:00:00 2001 From: Josue-T Date: Wed, 13 Feb 2019 22:17:32 +0100 Subject: [PATCH 074/113] Reload fail2ban instead of restart --- src/yunohost/firewall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py index 1c44efe99..9d209dbb8 100644 --- a/src/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -195,6 +195,7 @@ def firewall_reload(skip_upnp=False): """ from yunohost.hook import hook_callback + from yunohost.service import _run_service_command reloaded = False errors = False @@ -276,8 +277,7 @@ def firewall_reload(skip_upnp=False): # Refresh port forwarding with UPnP firewall_upnp(no_refresh=False) - # TODO: Use service_restart - os.system("service fail2ban restart") + _run_service_command("reload", "fail2ban") if errors: logger.warning(m18n.n('firewall_rules_cmd_failed')) From bba393c45c5ee45a942f6c7be697e7a2415e14b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Fri, 15 Feb 2019 14:06:53 +0100 Subject: [PATCH 075/113] Add possibility to use template or predefined config --- data/helpers.d/backend | 70 +++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index ac4d4ed94..c16da8f95 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -185,11 +185,18 @@ ynh_remove_fpm_config () { } # Create a dedicated fail2ban config (jail and filter conf files) +# usage 1: ynh_add_fail2ban_config log_file filter [max_retry [ports]] +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https # -# usage: ynh_add_fail2ban_config "list of others variables to replace" +# ----------------------------------------------------------------------------- # -# | arg: list of others variables to replace separeted by a space -# | for example : 'var_1 var_2 ...' +# usage 2: ynh_add_fail2ban_config -t [-v "list of others variables to replace"] +# | arg: -t, --use_template - Use this helper in template mode +# | arg: -v, --others_var= - List of others variables to replace separeted by a space +# | for example : 'var_1 var_2 ...' # # This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf # __APP__ by $app @@ -235,7 +242,16 @@ ynh_remove_fpm_config () { # To validate your regex you can test with this command: # fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf ynh_add_fail2ban_config () { - local others_var=${1:-} + # Declare an array to define the options of this helper. + declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) + local logpath + local failregex + local max_retry + local ports + local others_var + local use_template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" @@ -245,18 +261,42 @@ ynh_add_fail2ban_config () { cp ../conf/f2b_jail.conf $finalfail2banjailconf cp ../conf/f2b_filter.conf $finalfail2banfilterconf - if test -n "${app:-}"; then - ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" - ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" - fi + if [[ ${use_template:-0} == 1 ]]; then + if test -n "${app:-}"; then + ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" + ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" + fi - # Replace all other variable given as arguments - for var_to_replace in $others_var; do - # ${var_to_replace^^} make the content of the variable on upper-cases - # ${!var_to_replace} get the content of the variable named $var_to_replace - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" - done + # Replace all other variable given as arguments + for var_to_replace in ${others_var:-}; do + # ${var_to_replace^^} make the content of the variable on upper-cases + # ${!var_to_replace} get the content of the variable named $var_to_replace + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" + done + else + max_retry=${max_retry:-3} + ports=${ports:-http,https} + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + tee $finalfail2banjailconf < Date: Fri, 15 Feb 2019 16:18:36 +0100 Subject: [PATCH 076/113] Update usage comment --- data/helpers.d/backend | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index c16da8f95..c0722a5ce 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -185,15 +185,15 @@ ynh_remove_fpm_config () { } # Create a dedicated fail2ban config (jail and filter conf files) -# usage 1: ynh_add_fail2ban_config log_file filter [max_retry [ports]] +# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] # | arg: -l, --logpath= - Log file to be checked by fail2ban # | arg: -r, --failregex= - Failregex to be looked for by fail2ban # | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https # # ----------------------------------------------------------------------------- # -# usage 2: ynh_add_fail2ban_config -t [-v "list of others variables to replace"] +# usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"] # | arg: -t, --use_template - Use this helper in template mode # | arg: -v, --others_var= - List of others variables to replace separeted by a space # | for example : 'var_1 var_2 ...' From 54eb3efec0dce598cab7cc21446cda8485af25dc Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 15 Feb 2019 16:21:30 +0100 Subject: [PATCH 077/113] Reorganize comment header --- data/helpers.d/backend | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index c0722a5ce..a8da55ef7 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -185,6 +185,7 @@ ynh_remove_fpm_config () { } # Create a dedicated fail2ban config (jail and filter conf files) +# # usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] # | arg: -l, --logpath= - Log file to be checked by fail2ban # | arg: -r, --failregex= - Failregex to be looked for by fail2ban @@ -205,17 +206,6 @@ ynh_remove_fpm_config () { # __VAR_1__ by $var_1 # __VAR_2__ by $var_2 # -# Note about the "failregex" option: -# regex to match the password failure messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) -# -# You can find some more explainations about how to make a regex here : -# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters -# -# Note that the logfile need to exist before to call this helper !! -# # Generally your template will look like that by example (for synapse): # # f2b_jail.conf: @@ -239,8 +229,22 @@ ynh_remove_fpm_config () { # # ignoreregex = # +# ----------------------------------------------------------------------------- +# +# Note about the "failregex" option: +# regex to match the password failure messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# Note that the logfile need to exist before to call this helper !! +# # To validate your regex you can test with this command: # fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf +# ynh_add_fail2ban_config () { # Declare an array to define the options of this helper. declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) From ccb0cad296c335b63f9691e234c19ef2a2e11211 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 15 Feb 2019 16:32:13 +0100 Subject: [PATCH 078/113] Fix various issues --- data/helpers.d/backend | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index a8da55ef7..a7d3a09a1 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -256,17 +256,23 @@ ynh_add_fail2ban_config () { local use_template # Manage arguments with getopts ynh_handle_getopts_args "$@" + use_template="${use_template:-0}" + max_retry=${max_retry:-3} + ports=${ports:-http,https} finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" ynh_backup_if_checksum_is_different "$finalfail2banjailconf" ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" - cp ../conf/f2b_jail.conf $finalfail2banjailconf - cp ../conf/f2b_filter.conf $finalfail2banfilterconf + if [ $use_template -eq 1 ] + then + # Usage 2, templates + cp ../conf/f2b_jail.conf $finalfail2banjailconf + cp ../conf/f2b_filter.conf $finalfail2banfilterconf - if [[ ${use_template:-0} == 1 ]]; then - if test -n "${app:-}"; then + if [ -n "${app:-}" ] + then ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" fi @@ -278,9 +284,9 @@ ynh_add_fail2ban_config () { ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" done + else - max_retry=${max_retry:-3} - ports=${ports:-http,https} + # Usage 1, no template. Build a config file from scratch. test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." @@ -302,6 +308,7 @@ ignoreregex = EOF fi + # Common to usage 1 and 2. ynh_store_file_checksum "$finalfail2banjailconf" ynh_store_file_checksum "$finalfail2banfilterconf" @@ -309,8 +316,8 @@ EOF local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then - echo "[ERR] Fail2ban failed to load the jail for $app" >&2 - echo "WARNING${fail2ban_error#*WARNING}" >&2 + ynh_print_err --message="Fail2ban failed to load the jail for $app" + ynh_print_warn --message="${fail2ban_error#*WARNING}" fi } From 0e3f3159cb23735b3971ad5d0d5f531de03c3bf3 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 16 Feb 2019 00:20:51 +0100 Subject: [PATCH 079/113] ynh_print_warn instead of info --- data/helpers.d/filesystem | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 713c3890b..10123dea4 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -364,7 +364,7 @@ ynh_backup_if_checksum_is_different () { backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" sudo mkdir -p "$(dirname "$backup_file_checksum")" sudo cp -a "$file" "$backup_file_checksum" # Backup the current file - ynh_print_info "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" echo "$backup_file_checksum" # Return the name of the backup file fi fi From b62d72f77b31b9055e0a97aef68d3e2f19db30c9 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 16 Feb 2019 00:28:27 +0100 Subject: [PATCH 080/113] Factorize into ynh_read_manifest --- data/helpers.d/system | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 39af0092f..052ea5cec 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -67,6 +67,11 @@ ynh_read_manifest () { # Manage arguments with getopts ynh_handle_getopts_args "$@" + if [ ! -e "$manifest" ]; then + # If the manifest isn't found, try the common place for backup and restore script. + manifest="../settings/manifest.json" + fi + jq ".$manifest_key" "$manifest" --raw-output } @@ -85,9 +90,6 @@ ynh_app_upstream_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest" ]; then - manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place - fi version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/~ynh*/}" } @@ -107,9 +109,6 @@ ynh_app_package_version () { ynh_handle_getopts_args "$@" manifest="${manifest:-../manifest.json}" - if [ ! -e "$manifest" ]; then - manifest="../settings/manifest.json" # Into the restore script, the manifest is not at the same place - fi version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") echo "${version_key/*~ynh/}" } From 562b3b98f6817f6213dada93d24c2a7358d86b69 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 14:37:36 +0100 Subject: [PATCH 081/113] Improve messages when app upgrades start --- locales/en.json | 3 ++- src/yunohost/app.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index 8528c2576..24841311a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -36,7 +36,8 @@ "app_sources_fetch_failed": "Unable to fetch sources files", "app_unknown": "Unknown app", "app_unsupported_remote_type": "Unsupported remote type used for the app", - "app_upgrade_app_name": "Upgrading app {app}…", + "app_upgrade_several_apps": "The following apps will be upgraded : {apps}", + "app_upgrade_app_name": "Now upgrading app {app}…", "app_upgrade_failed": "Unable to upgrade {app:s}", "app_upgrade_some_app_failed": "Unable to upgrade some applications", "app_upgraded": "{app:s} has been upgraded", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0bca68787..23b4b49b7 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -586,7 +586,10 @@ def app_upgrade(auth, app=[], url=None, file=None): elif not isinstance(app, list): apps = [app] - logger.info("Upgrading apps %s", ", ".join(app)) + if len(apps) == 0: + raise YunohostError('app_no_upgrade') + if len(apps) > 1: + logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(app))) for app_instance_name in apps: logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name)) From 7d3319332dbfd3e3ac1527b82fbb763261c2554b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 14:38:39 +0100 Subject: [PATCH 082/113] Improve message when some upgrades fail --- locales/en.json | 3 ++- src/yunohost/app.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/locales/en.json b/locales/en.json index 24841311a..3681fd847 100644 --- a/locales/en.json +++ b/locales/en.json @@ -24,7 +24,8 @@ "app_location_install_failed": "Unable to install the app in this location because it conflit with the app '{other_app}' already installed on '{other_path}'", "app_location_unavailable": "This url is not available or conflicts with the already installed app(s):\n{apps:s}", "app_manifest_invalid": "Invalid app manifest: {error}", - "app_no_upgrade": "No app to upgrade", + "app_no_upgrade": "No apps to upgrade", + "app_not_upgraded": "The following apps were not upgraded: {apps}", "app_not_correctly_installed": "{app:s} seems to be incorrectly installed", "app_not_installed": "{app:s} is not installed", "app_not_properly_removed": "{app:s} has not been properly removed", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 23b4b49b7..609bece5e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -575,6 +575,7 @@ def app_upgrade(auth, app=[], url=None, file=None): raise YunohostError('app_no_upgrade') upgraded_apps = [] + not_upgraded_apps = [] apps = app user_specified_list = True @@ -651,6 +652,7 @@ def app_upgrade(auth, app=[], url=None, file=None): if hook_exec(extracted_app_folder + '/scripts/upgrade', args=args_list, env=env_dict) != 0: msg = m18n.n('app_upgrade_failed', app=app_instance_name) + not_upgraded_apps.append(app_instance_name) logger.error(msg) operation_logger.error(msg) else: @@ -684,8 +686,8 @@ def app_upgrade(auth, app=[], url=None, file=None): hook_callback('post_app_upgrade', args=args_list, env=env_dict) operation_logger.success() - if not upgraded_apps: - raise YunohostError('app_no_upgrade') + if not_upgraded_apps: + raise YunohostError('app_not_upgraded', apps=', '.join(not_upgraded_apps)) app_ssowatconf(auth) From fe37acd315749be005c95f7f0a107b9d8a2afdd2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 14:45:05 +0100 Subject: [PATCH 083/113] Avoid checking for duplicates --- src/yunohost/app.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 609bece5e..5e7f17b6e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -574,7 +574,6 @@ def app_upgrade(auth, app=[], url=None, file=None): except YunohostError: raise YunohostError('app_no_upgrade') - upgraded_apps = [] not_upgraded_apps = [] apps = app @@ -587,6 +586,9 @@ def app_upgrade(auth, app=[], url=None, file=None): elif not isinstance(app, list): apps = [app] + # Remove possible duplicates + apps = [app for i,app in enumerate(apps) if apps not in L[:i]] + if len(apps) == 0: raise YunohostError('app_no_upgrade') if len(apps) > 1: @@ -598,9 +600,6 @@ def app_upgrade(auth, app=[], url=None, file=None): if not installed: raise YunohostError('app_not_installed', app=app_instance_name) - if app_instance_name in upgraded_apps: - continue - app_dict = app_info(app_instance_name, raw=True) if file: @@ -680,7 +679,6 @@ def app_upgrade(auth, app=[], url=None, file=None): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) # So much win - upgraded_apps.append(app_instance_name) logger.success(m18n.n('app_upgraded', app=app_instance_name)) hook_callback('post_app_upgrade', args=args_list, env=env_dict) From d495d315c0e1c9492e94e0e810a0e6cdb3d4c975 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 16:36:35 +0100 Subject: [PATCH 084/113] [fix] Loading only one helper file leads to errors because missing getopts --- data/hooks/backup/05-conf_ldap | 2 +- data/hooks/backup/08-conf_ssh | 2 +- data/hooks/backup/11-conf_ynh_mysql | 2 +- data/hooks/backup/14-conf_ssowat | 2 +- data/hooks/backup/17-data_home | 2 +- data/hooks/backup/20-conf_ynh_firewall | 2 +- data/hooks/backup/21-conf_ynh_certs | 2 +- data/hooks/backup/23-data_mail | 2 +- data/hooks/backup/26-conf_xmpp | 2 +- data/hooks/backup/29-conf_nginx | 2 +- data/hooks/backup/32-conf_cron | 2 +- data/hooks/backup/40-conf_ynh_currenthost | 2 +- data/hooks/conf_regen/03-ssh | 2 +- data/hooks/conf_regen/15-nginx | 2 +- data/hooks/conf_regen/34-mysql | 4 +--- data/hooks/conf_regen/43-dnsmasq | 4 +--- data/hooks/restore/11-conf_ynh_mysql | 4 ++-- 17 files changed, 18 insertions(+), 22 deletions(-) diff --git a/data/hooks/backup/05-conf_ldap b/data/hooks/backup/05-conf_ldap index b21103ede..9ae22095e 100755 --- a/data/hooks/backup/05-conf_ldap +++ b/data/hooks/backup/05-conf_ldap @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ldap" diff --git a/data/hooks/backup/08-conf_ssh b/data/hooks/backup/08-conf_ssh index ae422617e..ee976080c 100755 --- a/data/hooks/backup/08-conf_ssh +++ b/data/hooks/backup/08-conf_ssh @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ssh" diff --git a/data/hooks/backup/11-conf_ynh_mysql b/data/hooks/backup/11-conf_ynh_mysql index 60bd8c017..031707337 100755 --- a/data/hooks/backup/11-conf_ynh_mysql +++ b/data/hooks/backup/11-conf_ynh_mysql @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/mysql" diff --git a/data/hooks/backup/14-conf_ssowat b/data/hooks/backup/14-conf_ssowat index ca42d3369..d4db72493 100755 --- a/data/hooks/backup/14-conf_ssowat +++ b/data/hooks/backup/14-conf_ssowat @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ssowat" diff --git a/data/hooks/backup/17-data_home b/data/hooks/backup/17-data_home index f7a797b6b..af00d67e8 100755 --- a/data/hooks/backup/17-data_home +++ b/data/hooks/backup/17-data_home @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/home" diff --git a/data/hooks/backup/20-conf_ynh_firewall b/data/hooks/backup/20-conf_ynh_firewall index 4e08114e7..98be3eb09 100755 --- a/data/hooks/backup/20-conf_ynh_firewall +++ b/data/hooks/backup/20-conf_ynh_firewall @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/firewall" diff --git a/data/hooks/backup/21-conf_ynh_certs b/data/hooks/backup/21-conf_ynh_certs index f9687164d..a3912a995 100755 --- a/data/hooks/backup/21-conf_ynh_certs +++ b/data/hooks/backup/21-conf_ynh_certs @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/certs" diff --git a/data/hooks/backup/23-data_mail b/data/hooks/backup/23-data_mail index 618a0aafe..7fdc883fd 100755 --- a/data/hooks/backup/23-data_mail +++ b/data/hooks/backup/23-data_mail @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/mail" diff --git a/data/hooks/backup/26-conf_xmpp b/data/hooks/backup/26-conf_xmpp index 12300a00a..b55ad2bfc 100755 --- a/data/hooks/backup/26-conf_xmpp +++ b/data/hooks/backup/26-conf_xmpp @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/xmpp" diff --git a/data/hooks/backup/29-conf_nginx b/data/hooks/backup/29-conf_nginx index d900c7535..81e145e24 100755 --- a/data/hooks/backup/29-conf_nginx +++ b/data/hooks/backup/29-conf_nginx @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/nginx" diff --git a/data/hooks/backup/32-conf_cron b/data/hooks/backup/32-conf_cron index 2fea9f53f..063ec1a3f 100755 --- a/data/hooks/backup/32-conf_cron +++ b/data/hooks/backup/32-conf_cron @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/cron" diff --git a/data/hooks/backup/40-conf_ynh_currenthost b/data/hooks/backup/40-conf_ynh_currenthost index e4a684576..6a98fd0d2 100755 --- a/data/hooks/backup/40-conf_ynh_currenthost +++ b/data/hooks/backup/40-conf_ynh_currenthost @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh" diff --git a/data/hooks/conf_regen/03-ssh b/data/hooks/conf_regen/03-ssh index 9de527518..5bb9cf916 100755 --- a/data/hooks/conf_regen/03-ssh +++ b/data/hooks/conf_regen/03-ssh @@ -2,7 +2,7 @@ set -e -. /usr/share/yunohost/helpers.d/utils +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 461c10c0c..7ca63c003 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -2,7 +2,7 @@ set -e -. /usr/share/yunohost/helpers.d/utils +. /usr/share/yunohost/helpers do_init_regen() { if [[ $EUID -ne 0 ]]; then diff --git a/data/hooks/conf_regen/34-mysql b/data/hooks/conf_regen/34-mysql index 5ee91827b..9f35fec18 100755 --- a/data/hooks/conf_regen/34-mysql +++ b/data/hooks/conf_regen/34-mysql @@ -2,6 +2,7 @@ set -e MYSQL_PKG="mariadb-server-10.1" +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 @@ -15,7 +16,6 @@ do_post_regen() { regen_conf_files=$1 if [ ! -f /etc/yunohost/mysql ]; then - . /usr/share/yunohost/helpers.d/string # ensure that mysql is running sudo systemctl -q is-active mysql.service \ @@ -25,8 +25,6 @@ do_post_regen() { mysql_password=$(ynh_string_random 10) sudo mysqladmin -s -u root -pyunohost password "$mysql_password" || { if [ $FORCE -eq 1 ]; then - . /usr/share/yunohost/helpers.d/package - echo "It seems that you have already configured MySQL." \ "YunoHost needs to have a root access to MySQL to runs its" \ "applications, and is going to reset the MySQL root password." \ diff --git a/data/hooks/conf_regen/43-dnsmasq b/data/hooks/conf_regen/43-dnsmasq index 2c8ce797b..ed795c058 100755 --- a/data/hooks/conf_regen/43-dnsmasq +++ b/data/hooks/conf_regen/43-dnsmasq @@ -1,13 +1,11 @@ #!/bin/bash set -e +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 - # source ip helpers - . /usr/share/yunohost/helpers.d/ip - cd /usr/share/yunohost/templates/dnsmasq # create directory for pending conf diff --git a/data/hooks/restore/11-conf_ynh_mysql b/data/hooks/restore/11-conf_ynh_mysql index 0aaaccd54..1336a2cc2 100644 --- a/data/hooks/restore/11-conf_ynh_mysql +++ b/data/hooks/restore/11-conf_ynh_mysql @@ -1,6 +1,8 @@ backup_dir="$1/conf/ynh/mysql" MYSQL_PKG="mariadb-server-10.1" +. /usr/share/yunohost/helpers + # ensure that mysql is running service mysql status >/dev/null 2>&1 \ || service mysql start @@ -11,13 +13,11 @@ service mysql status >/dev/null 2>&1 \ new_pwd=$(sudo cat "${backup_dir}/root_pwd" || sudo cat "${backup_dir}/mysql") [ -z "$curr_pwd" ] && curr_pwd="yunohost" [ -z "$new_pwd" ] && { - . /usr/share/yunohost/helpers.d/string new_pwd=$(ynh_string_random 10) } # attempt to change it sudo mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || { - . /usr/share/yunohost/helpers.d/package echo "It seems that you have already configured MySQL." \ "YunoHost needs to have a root access to MySQL to runs its" \ From 683d62d0e5df9dd07ca569e48a4c96e9ca9856d8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 18:43:14 +0100 Subject: [PATCH 085/113] [microdecision] Fix interface with meltdown checker script, stdout contains weird debug messages when ran inside LXC :| --- src/yunohost/tools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index a220e21ca..b2fbf380c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -735,6 +735,14 @@ def _check_if_vulnerable_to_meltdown(): output, err = call.communicate() assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode + # If there are multiple lines, sounds like there was some messages + # in stdout that are not json >.> ... Try to get the actual json + # stuff which should be the last line + output = output.strip() + if "\n" in output: + logger.debug("Original meltdown checker output : %s" % output) + output = output.split("\n")[-1] + CVEs = json.loads(output) assert len(CVEs) == 1 assert CVEs[0]["NAME"] == "MELTDOWN" From d0c7603d9db1a406c203e97f95c405e6bb0d941a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 20:25:30 +0100 Subject: [PATCH 086/113] Wtf is wrong with you sudo :| Why don't you forward the damn DEBIAN_FRONTEND --- data/helpers.d/package | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/package b/data/helpers.d/package index 1b7c38c5c..e7aa3faf8 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -71,7 +71,7 @@ ynh_package_version() { # usage: ynh_apt update ynh_apt() { ynh_wait_dpkg_free - DEBIAN_FRONTEND=noninteractive sudo apt-get -y $@ + DEBIAN_FRONTEND=noninteractive apt-get -y $@ } # Update package index files @@ -150,7 +150,7 @@ ynh_package_install_from_equivs () { cp "$controlfile" "${TMPDIR}/control" (cd "$TMPDIR" equivs-build ./control 1> /dev/null - DEBIAN_FRONTEND=noninteractive dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) + dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1) ynh_package_install -f || ynh_die --message="Unable to install dependencies" [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR # Remove the temp dir. From fc00b59be4561ab04009c87c49b60ddfe99f9189 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 20:44:50 +0100 Subject: [PATCH 087/113] Add messages before app install, remove, and some messages for backups --- locales/en.json | 7 ++++++- src/yunohost/app.py | 4 ++++ src/yunohost/backup.py | 6 +++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index 3681fd847..34258187e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -35,6 +35,10 @@ "app_requirements_failed": "Unable to meet requirements for {app}: {error}", "app_requirements_unmeet": "Requirements are not met for {app}, the package {pkgname} ({version}) must be {spec}", "app_sources_fetch_failed": "Unable to fetch sources files", + "app_start_install": "Installing application {app}…", + "app_start_remove": "Removing application {app}…", + "app_start_backup": "Collecting files to be backuped for {app}…", + "app_start_restore": "Restoring application {app}…", "app_unknown": "Unknown app", "app_unsupported_remote_type": "Unsupported remote type used for the app", "app_upgrade_several_apps": "The following apps will be upgraded : {apps}", @@ -65,6 +69,7 @@ "ask_path": "Path", "backup_abstract_method": "This backup method hasn't yet been implemented", "backup_action_required": "You must specify something to save", + "backup_actually_backuping": "Now creating a backup archive from the files collected…", "backup_app_failed": "Unable to back up the app '{app:s}'", "backup_applying_method_borg": "Sending all files to backup into borg-backup repository…", "backup_applying_method_copy": "Copying all files to backup…", @@ -101,6 +106,7 @@ "backup_method_copy_finished": "Backup copy finished", "backup_method_custom_finished": "Custom backup method '{method:s}' finished", "backup_method_tar_finished": "Backup tar archive created", + "backup_mount_archive_for_restore": "Preparing archive for restoration…", "backup_no_uncompress_archive_dir": "Uncompress archive directory doesn't exist", "backup_nothings_done": "There is nothing to save", "backup_output_directory_forbidden": "Forbidden output directory. Backups can't be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders", @@ -108,7 +114,6 @@ "backup_output_directory_required": "You must provide an output directory for the backup", "backup_output_symlink_dir_broken": "You have a broken symlink instead of your archives directory '{path:s}'. You may have a specific setup to backup your data on an other filesystem, in this case you probably forgot to remount or plug your hard dirve or usb key.", "backup_php5_to_php7_migration_may_fail": "Could not convert your archive to support php7, your php apps may fail to restore (reason: {error:s})", - "backup_running_app_script": "Running backup script of app '{app:s}'…", "backup_running_hooks": "Running backup hooks…", "backup_system_part_failed": "Unable to backup the '{part:s}' system part", "backup_unable_to_organize_files": "Unable to organize files in the archive with the quick method", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 5e7f17b6e..f5746fd4b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -801,6 +801,8 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on operation_logger.related_to.append(("app", app_id)) operation_logger.start() + logger.info(m18n.n("app_start_install", app=app_id)) + # Create app directory app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) if os.path.exists(app_setting_path): @@ -921,6 +923,8 @@ def app_remove(operation_logger, auth, app): operation_logger.start() + logger.info(m18n.n("app_start_remove", app=app)) + app_setting_path = APPS_SETTING_PATH + app # TODO: display fail messages from script diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index ed7799fc1..36d732114 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -668,7 +668,7 @@ class BackupManager(): tmp_app_bkp_dir = env_dict["YNH_APP_BACKUP_DIR"] settings_dir = os.path.join(self.work_dir, 'apps', app, 'settings') - logger.debug(m18n.n('backup_running_app_script', app=app)) + logger.info(m18n.n("app_start_backup", app=app)) try: # Prepare backup directory for the app filesystem.mkdir(tmp_app_bkp_dir, 0o750, True, uid='admin') @@ -1242,6 +1242,8 @@ class RestoreManager(): operation_logger = OperationLogger('backup_restore_app', related_to) operation_logger.start() + logger.info(m18n.n("app_start_restore", app=app_instance_name)) + # Check if the app is not already installed if _is_installed(app_instance_name): logger.error(m18n.n('restore_already_installed_app', @@ -2059,6 +2061,7 @@ def backup_create(name=None, description=None, methods=[], backup_manager.collect_files() # Apply backup methods on prepared files + logger.info(m18n.n("backup_actually_backuping")) backup_manager.backup() logger.success(m18n.n('backup_created')) @@ -2127,6 +2130,7 @@ def backup_restore(auth, name, system=[], apps=[], force=False): # Mount the archive then call the restore for each system part / app # # + logger.info(m18n.n("backup_mount_archive_for_restore")) restore_manager.mount() restore_manager.restore() From 1d8d3c282861d3889826d0db493bedf3a2aa1f06 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 18 Feb 2019 15:58:05 +0100 Subject: [PATCH 088/113] Add small helper to check if dpkg is in a broken state --- src/yunohost/utils/packages.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index 5ef97618b..9cb3bd974 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -19,6 +19,7 @@ """ import re +import os import logging from collections import OrderedDict @@ -470,3 +471,11 @@ def ynh_packages_version(*args, **kwargs): 'yunohost', 'yunohost-admin', 'moulinette', 'ssowat', with_repo=True ) + + +def dpkg_is_broken(): + # If dpkg is broken, /var/lib/dpkg/updates + # will contains files like 0001, 0002, ... + # ref: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + return any(re.match("^[0-9]+$", f) + for f in os.listdir("/var/lib/dpkg/updates/")) From 19bd4da10400497bfa8de70d22d4c105ad4e6b5a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 18 Feb 2019 15:58:24 +0100 Subject: [PATCH 089/113] Assert that dpkg is not broken when trying to install an app --- locales/en.json | 1 + src/yunohost/app.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/locales/en.json b/locales/en.json index 8528c2576..477563c8b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -146,6 +146,7 @@ "diagnosis_monitor_network_error": "Can't monitor network: {error}", "diagnosis_monitor_system_error": "Can't monitor system: {error}", "diagnosis_no_apps": "No installed application", + "dpkg_is_broken": "You cannot do this right now because dpkg/apt (the system package managers) seems to be in a broken state... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dnsmasq_isnt_installed": "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Unable to generate certificate", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0bca68787..d332971c4 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -722,6 +722,9 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on }, } + if packages.dpkg_is_broken(): + raise YunohostError(m18n.n("dpkg_is_broken")) + def confirm_install(confirm): # Ignore if there's nothing for confirm (good quality app), if --force is used From 9a7dc6d43e63fe776e5d4053786595a97c4aabda Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 19 Feb 2019 16:40:47 +0100 Subject: [PATCH 090/113] Be able to define hook to trigger after changing a setting value --- src/yunohost/settings.py | 51 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index bbfb3ca56..cb98fa57e 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -115,10 +115,18 @@ def settings_set(key, value): raise YunohostError('global_settings_unknown_type', setting=key, unknown_type=key_type) + old_value = settings[key].get("value") settings[key]["value"] = value - _save_settings(settings) + # TODO : whatdo if the old value is the same as + # the new value... + try: + trigger_post_change_hook(key, old_value, value) + except Exception as e: + logger.error("Post-change hook for setting %s failed : %s" % (key, e)) + raise + def settings_reset(key): """ @@ -235,3 +243,44 @@ def _save_settings(settings, location=SETTINGS_PATH): settings_fd.write(result) except Exception as e: raise YunohostError('global_settings_cant_write_settings', reason=e) + + +# Meant to be a dict of setting_name -> function to call +post_change_hooks = {} + + +def post_change_hook(setting_name): + def decorator(func): + assert setting_name in DEFAULTS.keys(), "The setting %s does not exists" % setting_name + assert setting_name not in post_change_hooks, "You can only register one post change hook per setting (in particular for %s)" % setting_name + post_change_hooks[setting_name] = func + return func + return decorator + + +def trigger_post_change_hook(setting_name, old_value, new_value): + if setting_name not in post_change_hooks: + logger.debug("Nothing to do after changing setting %s" % setting_name) + return + + f = post_change_hooks[setting_name] + f(old_value, new_value) + + +# =========================================== +# +# Actions to trigger when changing a setting +# You can define such an action with : +# +# @post_change_hook("your.setting.name") +# def some_function_name(old_value, new_value): +# # Do some stuff +# +# =========================================== + + +#@post_change_hook("example.int") +#def myfunc(old_value, new_value): +# print("In hook") +# print(old_value) +# print(new_value) From 39ae73604400ecb050af5c8df3d26d8180f236f9 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 19 Feb 2019 17:43:33 +0100 Subject: [PATCH 091/113] Ensure the tar file is closed --- src/yunohost/backup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index ed7799fc1..9268c7613 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1802,10 +1802,11 @@ class TarBackupMethod(BackupMethod): # Add the "source" into the archive and transform the path into # "dest" tar.add(path['source'], arcname=path['dest']) - tar.close() except IOError: logger.error(m18n.n('backup_archive_writing_error'), exc_info=1) raise YunohostError('backup_creation_failed') + finally: + tar.close() # Move info file shutil.copy(os.path.join(self.work_dir, 'info.json'), From a69cd443aece34d7225cb281acc2019583980985 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 19 Feb 2019 17:49:57 +0100 Subject: [PATCH 092/113] More explicit error --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 8528c2576..244eba80f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -75,7 +75,7 @@ "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'", "backup_archive_open_failed": "Unable to open the backup archive", "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup", - "backup_archive_writing_error": "Unable to add files to backup into the compressed archive", + "backup_archive_writing_error": "Unable to add files '{source:s}' (named in the archive: '{dest:s}') to backup into the compressed archive '{archive:s}'", "backup_ask_for_copying_if_needed": "Some files couldn't be prepared to be backuped using the method that avoid to temporarily waste space on the system. To perform the backup, {size:s}MB should be used temporarily. Do you agree?", "backup_borg_not_implemented": "Borg backup method is not yet implemented", "backup_cant_mount_uncompress_archive": "Unable to mount in readonly mode the uncompress archive directory", From 44455cd433eaf529e40df921fef1c33de3505086 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 19 Feb 2019 17:50:12 +0100 Subject: [PATCH 093/113] More explicit error --- src/yunohost/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 9268c7613..062343a46 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1803,7 +1803,7 @@ class TarBackupMethod(BackupMethod): # "dest" tar.add(path['source'], arcname=path['dest']) except IOError: - logger.error(m18n.n('backup_archive_writing_error'), exc_info=1) + logger.error(m18n.n('backup_archive_writing_error', source=path['source'], archive=self._archive_file, dest=path['dest']), exc_info=1) raise YunohostError('backup_creation_failed') finally: tar.close() From ddf2b49d544f2b77aaaf0562396e6ca5075d4c67 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 19 Feb 2019 17:57:38 +0100 Subject: [PATCH 094/113] Cache results from meltdown checker --- src/yunohost/tools.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index b2fbf380c..88f202cc7 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -713,6 +713,23 @@ def tools_diagnosis(auth, private=False): def _check_if_vulnerable_to_meltdown(): # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754 + # We use a cache file to avoid re-running the script so many times, + # which can be expensive (up to around 5 seconds on ARM) + # and make the admin appear to be slow (c.f. the calls to diagnosis + # from the webadmin) + # + # The cache is in /tmp and shall disappear upon reboot + # *or* we compare it to dpkg.log modification time + # such that it's re-ran if there was package upgrades + # (e.g. from yunohost) + cache_file = "/tmp/yunohost-meltdown-diagnosis" + dpkg_log = "/var/log/dpkg.log" + print(os.path.exists(cache_file)) + if os.path.exists(cache_file): + if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log): + logger.debug("Using cached results for meltdown checker, from %s" % cache_file) + return read_json(cache_file)[0]["VULNERABLE"] + # script taken from https://github.com/speed47/spectre-meltdown-checker # script commit id is store directly in the script file_dir = os.path.split(__file__)[0] @@ -722,6 +739,7 @@ def _check_if_vulnerable_to_meltdown(): # example output from the script: # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}] try: + logger.debug("Running meltdown vulnerability checker") call = subprocess.Popen("bash %s --batch json --variant 3" % SCRIPT_PATH, shell=True, stdout=subprocess.PIPE, @@ -752,6 +770,8 @@ def _check_if_vulnerable_to_meltdown(): logger.warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e) raise Exception("Command output for failed meltdown check: '%s'" % output) + logger.debug("Writing results from meltdown checker to cache file, %s" % cache_file) + write_to_json(cache_file, CVEs) return CVEs[0]["VULNERABLE"] From b2a606987cc8224b7d129f53fe470f81235d68cb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 19 Feb 2019 18:05:09 +0100 Subject: [PATCH 095/113] It was a debug print indeed :D --- src/yunohost/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 88f202cc7..4a2101121 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -724,7 +724,6 @@ def _check_if_vulnerable_to_meltdown(): # (e.g. from yunohost) cache_file = "/tmp/yunohost-meltdown-diagnosis" dpkg_log = "/var/log/dpkg.log" - print(os.path.exists(cache_file)) if os.path.exists(cache_file): if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log): logger.debug("Using cached results for meltdown checker, from %s" % cache_file) From 3c33eb078d3f87c97b59ef7bdb1e51ddd3f2702e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 16 Feb 2019 18:43:14 +0100 Subject: [PATCH 096/113] [microdecision] Fix interface with meltdown checker script, stdout contains weird debug messages when ran inside LXC :| --- src/yunohost/tools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 915b63940..4d4f5fffe 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -735,6 +735,14 @@ def _check_if_vulnerable_to_meltdown(): output, err = call.communicate() assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode + # If there are multiple lines, sounds like there was some messages + # in stdout that are not json >.> ... Try to get the actual json + # stuff which should be the last line + output = output.strip() + if "\n" in output: + logger.debug("Original meltdown checker output : %s" % output) + output = output.split("\n")[-1] + CVEs = json.loads(output) assert len(CVEs) == 1 assert CVEs[0]["NAME"] == "MELTDOWN" From ce79dd8c8a3ca1f16dda46cfd749714e3eb1fc07 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 19 Feb 2019 19:12:59 +0100 Subject: [PATCH 097/113] Update changelog for 3.4.2.4 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3f3a6a5ef..7be4212fe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.4.2.4) stable; urgency=low + + - [fix] Meltdown vulnerability checker something outputing trash instead of pure json + + -- Alexandre Aubin Tue, 19 Feb 2019 19:11:38 +0000 + yunohost (3.4.2.3) stable; urgency=low - [fix] Admin password appearing in logs after logging in on webadmin From c7245e5db1babff3746411a595a3f480fe9e9940 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 20 Feb 2019 15:02:14 +0100 Subject: [PATCH 098/113] Add apt-transport-https to dependencies --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index b359d5ec4..685c194ba 100644 --- a/debian/control +++ b/debian/control @@ -13,7 +13,7 @@ Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 2.7.1), ssowat (>= 2.7.1) , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 - , glances + , glances, apt-transport-https , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq , ca-certificates, netcat-openbsd, iproute , mariadb-server, php-mysql | php-mysqlnd From c029ccb7ad80f415fb3831bf5fc9dce217d8ee97 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 21 Feb 2019 15:07:06 +0100 Subject: [PATCH 099/113] Also feed the setting name when calling setting hook --- src/yunohost/settings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index cb98fa57e..81ea46114 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -264,7 +264,7 @@ def trigger_post_change_hook(setting_name, old_value, new_value): return f = post_change_hooks[setting_name] - f(old_value, new_value) + f(setting_name, old_value, new_value) # =========================================== @@ -273,14 +273,15 @@ def trigger_post_change_hook(setting_name, old_value, new_value): # You can define such an action with : # # @post_change_hook("your.setting.name") -# def some_function_name(old_value, new_value): +# def some_function_name(setting_name, old_value, new_value): # # Do some stuff # # =========================================== #@post_change_hook("example.int") -#def myfunc(old_value, new_value): +#def myfunc(setting_name, old_value, new_value): # print("In hook") +# print(setting_name) # print(old_value) # print(new_value) From 5c5330be3da369e1a255bb305009d35738927a8e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 21 Feb 2019 15:35:49 +0100 Subject: [PATCH 100/113] Issue happening with some weird app instance name ... --- src/yunohost/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0bca68787..302049ed8 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2345,6 +2345,7 @@ def _parse_app_instance_name(app_instance_name): True """ match = re_app_instance_name.match(app_instance_name) + assert match, "Could not parse app instance name : %s" % app_instance_name appid = match.groupdict().get('appid') app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1 return (appid, app_instance_nb) From bca2af3391a815873a7b63084d941640e0dbadf0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 21 Feb 2019 23:37:19 +0100 Subject: [PATCH 101/113] [microdecision] I'm sick of those people who end up with app repo being added as app list and messing up everything -.- --- src/yunohost/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 302049ed8..5f5d9f8f9 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -97,6 +97,9 @@ def app_fetchlist(url=None, name=None): name -- Name of the list url -- URL of remote JSON list """ + if not url.endswith(".json"): + raise YunohostError("This is not a valid application list url. It should end with .json.") + # If needed, create folder where actual appslists are stored if not os.path.exists(REPO_PATH): os.makedirs(REPO_PATH) From 7342f894e5e2087e97ba0a9c2595ccb103e6e6ce Mon Sep 17 00:00:00 2001 From: yalh76 Date: Fri, 22 Feb 2019 14:28:08 +0100 Subject: [PATCH 102/113] add the arch argument to ynh_install_nodejs CF https://github.com/YunoHost/issues/issues/1311#issuecomment-466379366 --- data/helpers.d/nodejs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index c4332b60c..9b16f5729 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -64,8 +64,9 @@ ynh_install_nodejs () { # Declare an array to define the options of this helper. local legacy_args=n - declare -Ar args_array=( [n]=nodejs_version= ) + declare -Ar args_array=( [n]=nodejs_version= [a]=arch=) local nodejs_version + local arch # Manage arguments with getopts ynh_handle_getopts_args "$@" @@ -98,7 +99,12 @@ ynh_install_nodejs () { test -x /usr/bin/npm_n && mv /usr/bin/npm_n /usr/bin/npm # Install the requested version of nodejs - n $nodejs_version + if [ -z "$arch" ] + then + n $nodejs_version + else + n $nodejs_version --arch=$arch + fi # Find the last "real" version for this major version of node. real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1) From 53ba867f30349b953216302badf491e99a93dea4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 14:46:53 +0100 Subject: [PATCH 103/113] Also forbid to app_upgrade if dpkg is broken --- src/yunohost/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d332971c4..05d5b1e4c 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -564,6 +564,9 @@ def app_upgrade(auth, app=[], url=None, file=None): url -- Git url to fetch for upgrade """ + if packages.dpkg_is_broken(): + raise YunohostError(m18n.n("dpkg_is_broken")) + from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback # Retrieve interface From e7241394afb33b0ddb46531addff8727a17d3b4b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 15:00:18 +0100 Subject: [PATCH 104/113] Adding dpkg checks after removal of an application --- src/yunohost/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 05d5b1e4c..61b9c69e4 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -880,6 +880,9 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on app_ssowatconf(auth) + if packages.dpkg_is_broken(): + logger.error(m18n.n("this_action_broke_dpkg")) + if install_retcode == -1: msg = m18n.n('operation_interrupted') + " " + error_msg raise YunohostError(msg, raw_msg=True) @@ -962,6 +965,9 @@ def app_remove(operation_logger, auth, app): hook_remove(app) app_ssowatconf(auth) + if packages.dpkg_is_broken(): + raise YunohostError(m18n.n("this_action_broke_dpkg")) + def app_addaccess(auth, apps, users=[]): """ From c824f10cc29b98b551f042fde10a894f08791855 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 15:04:34 +0100 Subject: [PATCH 105/113] Also check for dpkg corruption before upgrading system or app... --- src/yunohost/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index b2fbf380c..254e0533c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -524,6 +524,10 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal ignore_packages -- Ignore APT packages upgrade """ + from yunohost.utils import packages + if packages.dpkg_is_broken(): + raise YunohostError(m18n.n("dpkg_is_broken")) + failure = False # Retrieve interface From dcf2ca8b550dae29e9085288bd0380e4ca463fd9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 15:06:11 +0100 Subject: [PATCH 106/113] Check dpkg status asap in app_install --- src/yunohost/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 61b9c69e4..159017c89 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -708,6 +708,9 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on no_remove_on_failure -- Debug option to avoid removing the app on a failed installation force -- Do not ask for confirmation when installing experimental / low-quality apps """ + if packages.dpkg_is_broken(): + raise YunohostError(m18n.n("dpkg_is_broken")) + from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback from yunohost.log import OperationLogger @@ -725,9 +728,6 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on }, } - if packages.dpkg_is_broken(): - raise YunohostError(m18n.n("dpkg_is_broken")) - def confirm_install(confirm): # Ignore if there's nothing for confirm (good quality app), if --force is used From ba7bdb8f142c0d94eeecb12ca9feb3ddc3759c0f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 15:07:09 +0100 Subject: [PATCH 107/113] No m18n.n needed in YunohostErrors --- src/yunohost/app.py | 6 +++--- src/yunohost/tools.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 159017c89..0bd8e412f 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -565,7 +565,7 @@ def app_upgrade(auth, app=[], url=None, file=None): """ if packages.dpkg_is_broken(): - raise YunohostError(m18n.n("dpkg_is_broken")) + raise YunohostError("dpkg_is_broken") from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback @@ -709,7 +709,7 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on force -- Do not ask for confirmation when installing experimental / low-quality apps """ if packages.dpkg_is_broken(): - raise YunohostError(m18n.n("dpkg_is_broken")) + raise YunohostError("dpkg_is_broken") from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback from yunohost.log import OperationLogger @@ -966,7 +966,7 @@ def app_remove(operation_logger, auth, app): app_ssowatconf(auth) if packages.dpkg_is_broken(): - raise YunohostError(m18n.n("this_action_broke_dpkg")) + raise YunohostError("this_action_broke_dpkg") def app_addaccess(auth, apps, users=[]): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 254e0533c..9a6c7ccba 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -526,7 +526,7 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal """ from yunohost.utils import packages if packages.dpkg_is_broken(): - raise YunohostError(m18n.n("dpkg_is_broken")) + raise YunohostError("dpkg_is_broken") failure = False From 2fec4a6c6415b7ade56f3d0227d09b3018ee59d9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 15:13:59 +0100 Subject: [PATCH 108/113] Implement second message, this_action_broke_dpkg --- locales/en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/locales/en.json b/locales/en.json index 477563c8b..5c87106ec 100644 --- a/locales/en.json +++ b/locales/en.json @@ -464,6 +464,7 @@ "ssowat_persistent_conf_write_error": "Error while saving SSOwat persistent configuration: {error:s}. Edit /etc/ssowat/conf.json.persistent file to fix the JSON syntax", "system_upgraded": "The system has been upgraded", "system_username_exists": "Username already exists in the system users", + "this_action_broke_dpkg": "This action broke dpkg/apt (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "An unexpected error occured: {error}", "unit_unknown": "Unknown unit '{unit:s}'", From 6fc6d6cfe6b6045de4251cb9dac01d2acae370f9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 15:16:07 +0100 Subject: [PATCH 109/113] Who knows, maybe this folder doesn't exist in some context --- src/yunohost/utils/packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index 9cb3bd974..e10de6493 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -477,5 +477,7 @@ def dpkg_is_broken(): # If dpkg is broken, /var/lib/dpkg/updates # will contains files like 0001, 0002, ... # ref: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + if not os.path.isdir("/var/lib/dpkg/updates/"): + return False return any(re.match("^[0-9]+$", f) for f in os.listdir("/var/lib/dpkg/updates/")) From 0a84e2f2261ba67d4cc9dc71b430576f80ec726d Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 22 Feb 2019 15:38:04 +0100 Subject: [PATCH 110/113] Default value for $arch --- data/helpers.d/nodejs | 1 + 1 file changed, 1 insertion(+) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 9b16f5729..f48685ece 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -69,6 +69,7 @@ ynh_install_nodejs () { local arch # Manage arguments with getopts ynh_handle_getopts_args "$@" + arch=${arch:-} # Create $n_install_dir mkdir -p "$n_install_dir" From 198919adc836c3f7041697c631275599dfc75d28 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Feb 2019 16:24:19 +0100 Subject: [PATCH 111/113] Don't add Strict-Transport-Security header in nginx conf if using a self-signed cert --- data/templates/nginx/server.tpl.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 0c221f188..43d38ca98 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -51,7 +51,9 @@ server { # Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners # https://wiki.mozilla.org/Security/Guidelines/Web_Security # https://observatory.mozilla.org/ - more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; + {% if domain_cert_ca != "Self-signed" %} + more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; + {% endif %} more_set_headers "Content-Security-Policy : upgrade-insecure-requests"; more_set_headers "Content-Security-Policy-Report-Only : default-src https: data: 'unsafe-inline' 'unsafe-eval'"; more_set_headers "X-Content-Type-Options : nosniff"; From 81178e0841d81cf07da56826bda4c16403269a1c Mon Sep 17 00:00:00 2001 From: yalh76 Date: Fri, 22 Feb 2019 17:55:42 +0100 Subject: [PATCH 112/113] remove arguments and let ynh_install_nodejs manage the issue --- data/helpers.d/nodejs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index f48685ece..f277eeef0 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -64,12 +64,10 @@ ynh_install_nodejs () { # Declare an array to define the options of this helper. local legacy_args=n - declare -Ar args_array=( [n]=nodejs_version= [a]=arch=) + declare -Ar args_array=( [n]=nodejs_version= ) local nodejs_version - local arch # Manage arguments with getopts ynh_handle_getopts_args "$@" - arch=${arch:-} # Create $n_install_dir mkdir -p "$n_install_dir" @@ -100,11 +98,11 @@ ynh_install_nodejs () { test -x /usr/bin/npm_n && mv /usr/bin/npm_n /usr/bin/npm # Install the requested version of nodejs - if [ -z "$arch" ] + if [[ $uname =~ aarch64 || $uname =~ arm64]] then - n $nodejs_version + n $nodejs_version --arch=arm64 else - n $nodejs_version --arch=$arch + n $nodejs_version fi # Find the last "real" version for this major version of node. From dd287173087e772d45482c47644479c6a336e40d Mon Sep 17 00:00:00 2001 From: yalh76 Date: Sat, 23 Feb 2019 01:23:02 +0100 Subject: [PATCH 113/113] Fix $uname --- data/helpers.d/nodejs | 1 + 1 file changed, 1 insertion(+) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index f277eeef0..61a1414ef 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -98,6 +98,7 @@ ynh_install_nodejs () { test -x /usr/bin/npm_n && mv /usr/bin/npm_n /usr/bin/npm # Install the requested version of nodejs + uname=$(uname -m) if [[ $uname =~ aarch64 || $uname =~ arm64]] then n $nodejs_version --arch=arm64