logs: misc ad-hoc tweaks to limit the noise in log sharing

helpers/helpers.v2.1.d/apt

@@ -16,15 +16,14 @@ YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true"
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_apt_install_dependencies() {
-    local dependencies=$@
+
-    # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See  below)
+    # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below)
-    dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')"
+    local dependencies="$(sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g' <<< "$@" | sed 's/|/ | /')"
-    local dependencies=${dependencies//|/ | }
     local version=$(ynh_read_manifest "version")
     local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps
 
     # Handle specific versions
-    if [[ "$dependencies" =~ [\<=\>] ]]; then
+    if grep '[<=>]' <<< "$dependencies"; then
         # Replace version specifications by relationships syntax
         # https://www.debian.org/doc/debian-policy/ch-relationships.html
         # Sed clarification
@@ -33,7 +32,7 @@ ynh_apt_install_dependencies() {
         # \+ matches one or more occurence of the previous characters, for >= or >>.
         # [^,]\+ matches all characters except ','
         # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)'
-        dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')"
+        dependencies="$(sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g' <<< "$dependencies")"
     fi
 
     # ############################## #
@@ -43,7 +42,7 @@ ynh_apt_install_dependencies() {
     # Check for specific php dependencies which requires sury
     # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni"
     # The (?<=php) syntax corresponds to lookbehind ;)
-    local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u)
+    local specific_php_version=$(grep -oP '(?<=php)[0-9.]+(?=-|\>|)' <<< "$dependencies" | sort -u)
 
     if [[ -n "$specific_php_version" ]]
     then
@@ -128,21 +127,22 @@ EOF
         # NB: this is in a subshell (though not sure why exactly not just use pushd/popd...)
         cd "$TMPDIR"
         # Install the fake package without its dependencies with dpkg --force-depends
-        LC_ALL=C equivs-build ./control 2>&1
+        LC_ALL=C equivs-build ./control > ./equivs_log 2>&1 || { cat ./equivs_log; false; }
-        LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" 2>&1 | tee ./dpkg_log
+        LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" > ./dpkg_log 2>&1
     )
 
     # Then install the missing dependencies with apt install
-    _ynh_apt_install --fix-broken \
+    _ynh_apt_install --fix-broken || {
-        || { # If the installation failed
+        # If the installation failed
-            # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
+        # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
-            # Parse the list of problematic dependencies from dpkg's log ...
+        # Parse the list of problematic dependencies from dpkg's log ...
-            # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
+        # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
-            local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')"
+        cat $TMPDIR/dpkg_log
-            # Fake an install of those dependencies to see the errors
+        local problematic_dependencies="$(grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' $TMPDIR/dpkg_log | tr '\n' ' ')"
-            # The sed command here is, Print only from 'Reading state info' to the end.
+        # Fake an install of those dependencies to see the errors
-            [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
+        # The sed command here is, Print only from 'Reading state info' to the end.
-            ynh_die "Unable to install dependencies"
+        [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
+        ynh_die "Unable to install dependencies"
     }
     rm --recursive --force "$TMPDIR" # Remove the temp dir.
 

helpers/helpers.v2.1.d/backup

@@ -211,6 +211,7 @@ _ynh_file_checksum_exists() {
 #
 # usage: ynh_store_file_checksum /path/to/file
 ynh_store_file_checksum() {
+    set +o xtrace # set +x
     local file=$1
     local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
 
@@ -231,6 +232,7 @@ ynh_store_file_checksum() {
     fi
     # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
     unset backup_file_checksum
+    set -o xtrace # set -x
 }
 
 # Verify the checksum and backup the file if it's different
@@ -240,6 +242,7 @@ ynh_store_file_checksum() {
 # This helper is primarily meant to allow to easily backup personalised/manually
 # modified config files.
 ynh_backup_if_checksum_is_different() {
+    set +o xtrace # set +x
     local file=$1
     local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
     local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name)
@@ -263,6 +266,7 @@ ynh_backup_if_checksum_is_different() {
             fi
         fi
     fi
+    set -o xtrace # set -x
 }
 
 # Delete a file checksum from the app settings

helpers/helpers.v2.1.d/getopts

@@ -46,10 +46,13 @@
 #
 # Requires YunoHost version 3.2.2 or higher.
 ynh_handle_getopts_args() {
+    # Trick to only re-enable debugging if it was set before
+    local xtrace_enable=$(set +o | grep xtrace)
+
     # Manage arguments only if there's some provided
     set +o xtrace # set +x
     if [ $# -eq 0 ]; then
-        set -o xtrace # set -x
+        eval "$xtrace_enable"
         return
     fi
 
@@ -181,5 +184,5 @@ ynh_handle_getopts_args() {
     # Call parse_arg and pass the modified list of args as an array of arguments.
     parse_arg "${arguments[@]}"
     
-    set -o xtrace # set -x
+    eval "$xtrace_enable"
 }

helpers/helpers.v2.1.d/php

@@ -176,6 +176,8 @@ ynh_config_remove_phpfpm() {
 #
 _ynh_get_scalable_phpfpm() {
 
+    set +o xtrace # set +x
+
     # If no usage provided, default to the value existing in setting ... or to low
     local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage)
     local usage=${fpm_usage_in_setting:-low}
@@ -195,16 +197,6 @@ _ynh_get_scalable_phpfpm() {
         footprint=50
     fi
 
-    # Define the factor to determine min_spare_servers
-    # to avoid having too few children ready to start for heavy apps
-    if [ $footprint -le 20 ]; then
-        min_spare_servers_factor=8
-    elif [ $footprint -le 35 ]; then
-        min_spare_servers_factor=5
-    else
-        min_spare_servers_factor=3
-    fi
-
     # Define the way the process manager handle child processes.
     if [ "$usage" = "low" ]; then
         php_pm=ondemand
@@ -216,9 +208,6 @@ _ynh_get_scalable_phpfpm() {
         ynh_die "Does not recognize '$usage' as an usage value."
     fi
 
-    # Get the total of RAM available, except swap.
-    local max_ram=$(ynh_get_ram --total)
-
     at_least_one() {
         # Do not allow value below 1
         if [ $1 -le 0 ]; then
@@ -228,10 +217,13 @@ _ynh_get_scalable_phpfpm() {
         fi
     }
 
+    # Get the total of RAM available, except swap.
+    local total_ram=$(ynh_get_ram --total)
+
     # Define pm.max_children
     # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app.
     # So if PHP-FPM start the maximum of children, it won't exceed half of the ram.
-    php_max_children=$(($max_ram / 2 / $footprint))
+    php_max_children=$(($total_ram / 2 / $footprint))
     # If process manager is set as static, use half less children.
     # Used as static, there's always as many children as the value of pm.max_children
     if [ "$php_pm" = "static" ]; then
@@ -253,6 +245,17 @@ _ynh_get_scalable_phpfpm() {
     fi
 
     if [ "$php_pm" = "dynamic" ]; then
+
+        # Define the factor to determine min_spare_servers
+        # to avoid having too few children ready to start for heavy apps
+        if [ $footprint -le 20 ]; then
+            min_spare_servers_factor=8
+        elif [ $footprint -le 35 ]; then
+            min_spare_servers_factor=5
+        else
+            min_spare_servers_factor=3
+        fi
+
         # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager
         php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor))
         php_min_spare_servers=$(at_least_one $php_min_spare_servers)
@@ -267,6 +270,15 @@ _ynh_get_scalable_phpfpm() {
         php_max_spare_servers=0
         php_start_servers=0
     fi
+
+    set -o xtrace # set -x
+
+    # For debugging, since otherwise things are hidden with set +x/-x
+    echo "php_pm:                $php_pm"
+    echo "php_max_children:      $php_max_children"
+    echo "php_min_spare_servers: $php_min_spare_servers"
+    echo "php_max_spare_servers: $php_max_spare_servers"
+    echo "php_start_servers:     $php_start_servers"
 }
 
 # Execute a command with Composer

helpers/helpers.v2.1.d/setting

@@ -68,6 +68,8 @@ ynh_app_setting_delete() {
 # [internal]
 #
 ynh_app_setting() {
+    # Trick to only re-enable debugging if it was set before
+    local xtrace_enable=$(set +o | grep xtrace)
     set +o xtrace # set +x
     ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <<EOF
 import os, yaml, sys
@@ -91,5 +93,5 @@ else:
     with open(setting_file, "w") as f:
         yaml.safe_dump(settings, f, default_flow_style=False)
 EOF
-    set -o xtrace # set -x
+    eval "$xtrace_enable"
 }

helpers/helpers.v2.1.d/systemd

@@ -114,7 +114,7 @@ ynh_systemctl() {
 
     # Start the timeout and try to find wait_until
     if [[ -n "${wait_until:-}" ]]; then
-        set +x
+        set +o xtrace # set +x
         local i=0
         local starttime=$(date +%s)
         for i in $(seq 1 $timeout); do
@@ -144,7 +144,7 @@ ynh_systemctl() {
             fi
             sleep 1
         done
-        set -x
+        set -o xtrace # set -x
         if [ $i -ge 3 ]; then
             echo "" >&2
         fi

helpers/helpers.v2.1.d/utils

@@ -158,7 +158,6 @@ ynh_setup_source() {
 
     local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')"
     local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')"
-    local src_sumprg="sha256sum"
     local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')"
     local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')"
     src_in_subdir=${src_in_subdir:-true}
@@ -215,7 +214,7 @@ ynh_setup_source() {
         [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?"
 
         # If the file was prefetched but somehow doesn't match the sum, rm and redownload it
-        if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
+        if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status
         then
             rm -f "$src_filename"
         fi
@@ -233,9 +232,9 @@ ynh_setup_source() {
         fi
 
         # Check the control sum
-        if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
+        if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status
         then
-            local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)"
+            local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)"
             local actual_size="$(du -hs ${src_filename} | cut --fields=1)"
             rm -f ${src_filename}
             ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})."

src/log.py

@@ -45,12 +45,18 @@ LOG_FILE_EXT = ".log"
 BORING_LOG_LINES = [
     r"set [+-]x$",
     r"set [+-]o xtrace$",
+    r"\+ set \+o$",
+    r"\+ grep xtrace$",
+    r"local 'xtrace_enable=",
     r"set [+-]o errexit$",
     r"set [+-]o nounset$",
     r"trap '' EXIT",
     r"local \w+$",
     r"local exit_code=(1|0)$",
     r"local legacy_args=.*$",
+    r"local _globalapp=.*$",
+    r"local checksum_setting_name=.*$",
+    r"ynh_app_setting ",  # (note the trailing space to match the "low level" one called by other setting helpers)
     r"local -A args_array$",
     r"args_array=.*$",
     r"ret_code=1",
@@ -62,8 +68,17 @@ BORING_LOG_LINES = [
     r"\[?\['? -n '' '?\]\]?$",
     r"rm -rf /var/cache/yunohost/download/$",
     r"type -t ynh_clean_setup$",
+    r"DEBUG - \+ unset \S+$",
     r"DEBUG - \+ echo '",
+    r"DEBUG - \+ LC_ALL=C$",
+    r"DEBUG - \+ DEBIAN_FRONTEND=noninteractive$",
     r"DEBUG - \+ exit (1|0)$",
+    r"DEBUG - \+ app=\S+$",
+    r"DEBUG - \+\+ app=\S+$",
+    r"DEBUG - \+\+ jq -r .\S+$",
+    r"DEBUG - \+\+ sed 's/\^null\$//'$",
+    "DEBUG - \\+ sed --in-place \$'s\\\\001",
+    "DEBUG - \\+ sed --in-place 's\u0001.*$",
 ]