diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 520007e48..a68b3b29d 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -16,15 +16,14 @@ YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true" # # Requires YunoHost version 2.6.4 or higher. ynh_apt_install_dependencies() { - local dependencies=$@ - # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) - dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" - local dependencies=${dependencies//|/ | } + + # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) + local dependencies="$(sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g' <<< "$@" | sed 's/|/ | /')" local version=$(ynh_read_manifest "version") local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps # Handle specific versions - if [[ "$dependencies" =~ [\<=\>] ]]; then + if grep '[<=>]' <<< "$dependencies"; then # Replace version specifications by relationships syntax # https://www.debian.org/doc/debian-policy/ch-relationships.html # Sed clarification @@ -33,7 +32,7 @@ ynh_apt_install_dependencies() { # \+ matches one or more occurence of the previous characters, for >= or >>. # [^,]\+ matches all characters except ',' # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)' - dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')" + dependencies="$(sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g' <<< "$dependencies")" fi # ############################## # @@ -43,7 +42,7 @@ ynh_apt_install_dependencies() { # Check for specific php dependencies which requires sury # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni" # The (?<=php) syntax corresponds to lookbehind ;) - local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u) + local specific_php_version=$(grep -oP '(?<=php)[0-9.]+(?=-|\>|)' <<< "$dependencies" | sort -u) if [[ -n "$specific_php_version" ]] then @@ -128,21 +127,22 @@ EOF # NB: this is in a subshell (though not sure why exactly not just use pushd/popd...) cd "$TMPDIR" # Install the fake package without its dependencies with dpkg --force-depends - LC_ALL=C equivs-build ./control 2>&1 - LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" 2>&1 | tee ./dpkg_log + LC_ALL=C equivs-build ./control > ./equivs_log 2>&1 || { cat ./equivs_log; false; } + LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" > ./dpkg_log 2>&1 ) # Then install the missing dependencies with apt install - _ynh_apt_install --fix-broken \ - || { # If the installation failed - # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) - # Parse the list of problematic dependencies from dpkg's log ... - # (relevant lines look like: "foo-ynh-deps depends on bar; however:") - local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')" - # Fake an install of those dependencies to see the errors - # The sed command here is, Print only from 'Reading state info' to the end. - [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 - ynh_die "Unable to install dependencies" + _ynh_apt_install --fix-broken || { + # If the installation failed + # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) + # Parse the list of problematic dependencies from dpkg's log ... + # (relevant lines look like: "foo-ynh-deps depends on bar; however:") + cat $TMPDIR/dpkg_log + local problematic_dependencies="$(grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' $TMPDIR/dpkg_log | tr '\n' ' ')" + # Fake an install of those dependencies to see the errors + # The sed command here is, Print only from 'Reading state info' to the end. + [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 + ynh_die "Unable to install dependencies" } rm --recursive --force "$TMPDIR" # Remove the temp dir. diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index abb41e041..c310cc0b8 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -211,6 +211,7 @@ _ynh_file_checksum_exists() { # # usage: ynh_store_file_checksum /path/to/file ynh_store_file_checksum() { + set +o xtrace # set +x local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' @@ -231,6 +232,7 @@ ynh_store_file_checksum() { fi # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. unset backup_file_checksum + set -o xtrace # set -x } # Verify the checksum and backup the file if it's different @@ -240,6 +242,7 @@ ynh_store_file_checksum() { # This helper is primarily meant to allow to easily backup personalised/manually # modified config files. ynh_backup_if_checksum_is_different() { + set +o xtrace # set +x local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) @@ -263,6 +266,7 @@ ynh_backup_if_checksum_is_different() { fi fi fi + set -o xtrace # set -x } # Delete a file checksum from the app settings diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index a0ef13b13..381c74256 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -46,10 +46,13 @@ # # Requires YunoHost version 3.2.2 or higher. ynh_handle_getopts_args() { + # Trick to only re-enable debugging if it was set before + local xtrace_enable=$(set +o | grep xtrace) + # Manage arguments only if there's some provided set +o xtrace # set +x if [ $# -eq 0 ]; then - set -o xtrace # set -x + eval "$xtrace_enable" return fi @@ -181,5 +184,5 @@ ynh_handle_getopts_args() { # Call parse_arg and pass the modified list of args as an array of arguments. parse_arg "${arguments[@]}" - set -o xtrace # set -x + eval "$xtrace_enable" } diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index dd252cd45..e8ff6d51d 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -176,6 +176,8 @@ ynh_config_remove_phpfpm() { # _ynh_get_scalable_phpfpm() { + set +o xtrace # set +x + # If no usage provided, default to the value existing in setting ... or to low local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) local usage=${fpm_usage_in_setting:-low} @@ -195,16 +197,6 @@ _ynh_get_scalable_phpfpm() { footprint=50 fi - # Define the factor to determine min_spare_servers - # to avoid having too few children ready to start for heavy apps - if [ $footprint -le 20 ]; then - min_spare_servers_factor=8 - elif [ $footprint -le 35 ]; then - min_spare_servers_factor=5 - else - min_spare_servers_factor=3 - fi - # Define the way the process manager handle child processes. if [ "$usage" = "low" ]; then php_pm=ondemand @@ -216,9 +208,6 @@ _ynh_get_scalable_phpfpm() { ynh_die "Does not recognize '$usage' as an usage value." fi - # Get the total of RAM available, except swap. - local max_ram=$(ynh_get_ram --total) - at_least_one() { # Do not allow value below 1 if [ $1 -le 0 ]; then @@ -228,10 +217,13 @@ _ynh_get_scalable_phpfpm() { fi } + # Get the total of RAM available, except swap. + local total_ram=$(ynh_get_ram --total) + # Define pm.max_children # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. # So if PHP-FPM start the maximum of children, it won't exceed half of the ram. - php_max_children=$(($max_ram / 2 / $footprint)) + php_max_children=$(($total_ram / 2 / $footprint)) # If process manager is set as static, use half less children. # Used as static, there's always as many children as the value of pm.max_children if [ "$php_pm" = "static" ]; then @@ -253,6 +245,17 @@ _ynh_get_scalable_phpfpm() { fi if [ "$php_pm" = "dynamic" ]; then + + # Define the factor to determine min_spare_servers + # to avoid having too few children ready to start for heavy apps + if [ $footprint -le 20 ]; then + min_spare_servers_factor=8 + elif [ $footprint -le 35 ]; then + min_spare_servers_factor=5 + else + min_spare_servers_factor=3 + fi + # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor)) php_min_spare_servers=$(at_least_one $php_min_spare_servers) @@ -267,6 +270,15 @@ _ynh_get_scalable_phpfpm() { php_max_spare_servers=0 php_start_servers=0 fi + + set -o xtrace # set -x + + # For debugging, since otherwise things are hidden with set +x/-x + echo "php_pm: $php_pm" + echo "php_max_children: $php_max_children" + echo "php_min_spare_servers: $php_min_spare_servers" + echo "php_max_spare_servers: $php_max_spare_servers" + echo "php_start_servers: $php_start_servers" } # Execute a command with Composer diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 0d737491e..84d755b3b 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -68,6 +68,8 @@ ynh_app_setting_delete() { # [internal] # ynh_app_setting() { + # Trick to only re-enable debugging if it was set before + local xtrace_enable=$(set +o | grep xtrace) set +o xtrace # set +x ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <&2 fi diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index e6eb2e88b..673ce6ba8 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -158,7 +158,6 @@ ynh_setup_source() { local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" - local src_sumprg="sha256sum" local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" src_in_subdir=${src_in_subdir:-true} @@ -215,7 +214,7 @@ ynh_setup_source() { [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" # If the file was prefetched but somehow doesn't match the sum, rm and redownload it - if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status then rm -f "$src_filename" fi @@ -233,9 +232,9 @@ ynh_setup_source() { fi # Check the control sum - if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status then - local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)" local actual_size="$(du -hs ${src_filename} | cut --fields=1)" rm -f ${src_filename} ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." diff --git a/src/log.py b/src/log.py index 818d0c1a9..45dc884fc 100755 --- a/src/log.py +++ b/src/log.py @@ -45,12 +45,18 @@ LOG_FILE_EXT = ".log" BORING_LOG_LINES = [ r"set [+-]x$", r"set [+-]o xtrace$", + r"\+ set \+o$", + r"\+ grep xtrace$", + r"local 'xtrace_enable=", r"set [+-]o errexit$", r"set [+-]o nounset$", r"trap '' EXIT", r"local \w+$", r"local exit_code=(1|0)$", r"local legacy_args=.*$", + r"local _globalapp=.*$", + r"local checksum_setting_name=.*$", + r"ynh_app_setting ", # (note the trailing space to match the "low level" one called by other setting helpers) r"local -A args_array$", r"args_array=.*$", r"ret_code=1", @@ -62,8 +68,17 @@ BORING_LOG_LINES = [ r"\[?\['? -n '' '?\]\]?$", r"rm -rf /var/cache/yunohost/download/$", r"type -t ynh_clean_setup$", + r"DEBUG - \+ unset \S+$", r"DEBUG - \+ echo '", + r"DEBUG - \+ LC_ALL=C$", + r"DEBUG - \+ DEBIAN_FRONTEND=noninteractive$", r"DEBUG - \+ exit (1|0)$", + r"DEBUG - \+ app=\S+$", + r"DEBUG - \+\+ app=\S+$", + r"DEBUG - \+\+ jq -r .\S+$", + r"DEBUG - \+\+ sed 's/\^null\$//'$", + "DEBUG - \\+ sed --in-place \$'s\\\\001", + "DEBUG - \\+ sed --in-place 's\u0001.*$", ]