Merge branch 'YunoHost:dev' into dev

debian/changelog

@@ -1,3 +1,24 @@
+yunohost (11.2.16) stable; urgency=low
+
+  - apps/logs: fix some information not being redacted because of the packaging v2 flows (a25033bb)
+  - logs: misc ad-hoc tweaks to limit the noise in log sharing (06c8fbc8)
+  - helpers: (1/2/2.1) add a new ynh_app_setting_set_default to replace the unecessarily complex 'if [ -z ${foo:-} ]' trick ([#1873](http://github.com/YunoHost/yunohost/pull/1873))
+  - helpers2.1: drop unused 'local source' mechanism from ynh_setup_source (dd8db188)
+  - helpers2.1: fix positional arg parsing in ynh_psql_create_user (e5585136)
+  - helpers2.1: rework the fpm usage/footprint madness ([#1874](http://github.com/YunoHost/yunohost/pull/1874))
+  - helpers2.1: fix ynh_config_add_logrotate when no arg is passed (3942ea12)
+  - helpers2.1: sudo -u$app -> sudo -u $app (d4857834)
+  - helpers2.1: change default timeout of ynh_systemctl to 60s instead of 300s (262453f1)
+  - helpers2.1: display 100 lines instead of 20 in CI context when service fails to start (9298738d)
+  - helpers2.1: when using ynh_systemctl to reload/start/restart a service with a wait_until and it timesout, handle it as a failure rather than keep going (b3409729)
+  - helpers2.1: for some reason sudo -E doesn't preserve PATH even though it's exported, so we gotta explicitly use --preserve-env=PATH (5f6df6a8)
+  - helpers2.1: var rename / cosmetic etc for nodejs/ruby/go version and install directories (2b1f7426)
+  - i18n: Translations updated for Basque, Slovak
+
+  Thanks to all contributors <3 ! (alexAubin, Jose Riha, xabirequejo)
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Sun, 23 Jun 2024 15:30:22 +0200
+
 yunohost (11.2.15) stable; urgency=low
 
   - apps: new experimentals "2.1" helpers ([#1855](http://github.com/YunoHost/yunohost/pull/1855))

helpers/helpers.v1.d/setting

@@ -52,6 +52,42 @@ ynh_app_setting_set() {
     fi
 }
 
+# Set an application setting but only if the "$key" variable ain't set yet
+#
+# Note that it doesn't just define the setting but ALSO define the $foobar variable
+#
+# Hence it's meant as a replacement for this legacy overly complex syntax:
+#
+# if [ -z "${foo:-}" ]
+# then
+#     foo="bar"
+#     ynh_app_setting_set --key="foo" --value="$foo"
+# fi
+#
+# usage: ynh_app_setting_set_default --app=app --key=key --value=value
+# | arg: -a, --app=     - the application id
+# | arg: -k, --key=     - the setting name to set
+# | arg: -v, --value=   - the default setting value to set
+#
+# Requires YunoHost version 11.1.16 or higher.
+ynh_app_setting_set_default() {
+    local _globalapp=${app-:}
+    # Declare an array to define the options of this helper.
+    local legacy_args=akv
+    local -A args_array=([a]=app= [k]=key= [v]=value=)
+    local app
+    local key
+    local value
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    app="${app:-$_globalapp}"
+
+    if [ -z "${!key:-}" ]; then
+        eval $key=\$value
+        ynh_app_setting "set" "$app" "$key" "$value"
+    fi
+}
+
 # Delete an application setting
 #
 # usage: ynh_app_setting_delete --app=app --key=key

helpers/helpers.v2.1.d/apt

@@ -16,15 +16,14 @@ YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true"
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_apt_install_dependencies() {
-    local dependencies=$@
+
-    # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See  below)
+    # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below)
-    dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')"
+    local dependencies="$(sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g' <<< "$@" | sed 's/|/ | /')"
-    local dependencies=${dependencies//|/ | }
     local version=$(ynh_read_manifest "version")
     local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps
 
     # Handle specific versions
-    if [[ "$dependencies" =~ [\<=\>] ]]; then
+    if grep '[<=>]' <<< "$dependencies"; then
         # Replace version specifications by relationships syntax
         # https://www.debian.org/doc/debian-policy/ch-relationships.html
         # Sed clarification
@@ -33,7 +32,7 @@ ynh_apt_install_dependencies() {
         # \+ matches one or more occurence of the previous characters, for >= or >>.
         # [^,]\+ matches all characters except ','
         # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)'
-        dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')"
+        dependencies="$(sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g' <<< "$dependencies")"
     fi
 
     # ############################## #
@@ -43,7 +42,7 @@ ynh_apt_install_dependencies() {
     # Check for specific php dependencies which requires sury
     # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni"
     # The (?<=php) syntax corresponds to lookbehind ;)
-    local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u)
+    local specific_php_version=$(grep -oP '(?<=php)[0-9.]+(?=-|\>|)' <<< "$dependencies" | sort -u)
 
     if [[ -n "$specific_php_version" ]]
     then
@@ -57,12 +56,9 @@ ynh_apt_install_dependencies() {
 
         # If the PHP version changed, remove the old fpm conf
         if [ -n "$old_php_version" ] && [ "$old_php_version" != "$specific_php_version" ]; then
-            local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir)
+            if [[ -f "/etc/php/$php_version/fpm/pool.d/$app.conf" ]]
-            local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
-
-            if [[ -f "$old_php_finalphpconf" ]]
             then
-                ynh_backup_if_checksum_is_different "$old_php_finalphpconf"
+                ynh_backup_if_checksum_is_different "/etc/php/$php_version/fpm/pool.d/$app.conf"
                 ynh_remove_fpm_config
             fi
         fi
@@ -128,21 +124,22 @@ EOF
         # NB: this is in a subshell (though not sure why exactly not just use pushd/popd...)
         cd "$TMPDIR"
         # Install the fake package without its dependencies with dpkg --force-depends
-        LC_ALL=C equivs-build ./control 2>&1
+        LC_ALL=C equivs-build ./control > ./equivs_log 2>&1 || { cat ./equivs_log; false; }
-        LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" 2>&1 | tee ./dpkg_log
+        LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" > ./dpkg_log 2>&1
     )
 
     # Then install the missing dependencies with apt install
-    _ynh_apt_install --fix-broken \
+    _ynh_apt_install --fix-broken || {
-        || { # If the installation failed
+        # If the installation failed
-            # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
+        # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
-            # Parse the list of problematic dependencies from dpkg's log ...
+        # Parse the list of problematic dependencies from dpkg's log ...
-            # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
+        # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
-            local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')"
+        cat $TMPDIR/dpkg_log
-            # Fake an install of those dependencies to see the errors
+        local problematic_dependencies="$(grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' $TMPDIR/dpkg_log | tr '\n' ' ')"
-            # The sed command here is, Print only from 'Reading state info' to the end.
+        # Fake an install of those dependencies to see the errors
-            [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
+        # The sed command here is, Print only from 'Reading state info' to the end.
-            ynh_die "Unable to install dependencies"
+        [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
+        ynh_die "Unable to install dependencies"
     }
     rm --recursive --force "$TMPDIR" # Remove the temp dir.
 

helpers/helpers.v2.1.d/backup

@@ -211,6 +211,7 @@ _ynh_file_checksum_exists() {
 #
 # usage: ynh_store_file_checksum /path/to/file
 ynh_store_file_checksum() {
+    set +o xtrace # set +x
     local file=$1
     local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
 
@@ -231,6 +232,7 @@ ynh_store_file_checksum() {
     fi
     # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
     unset backup_file_checksum
+    set -o xtrace # set -x
 }
 
 # Verify the checksum and backup the file if it's different
@@ -240,6 +242,7 @@ ynh_store_file_checksum() {
 # This helper is primarily meant to allow to easily backup personalised/manually
 # modified config files.
 ynh_backup_if_checksum_is_different() {
+    set +o xtrace # set +x
     local file=$1
     local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
     local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name)
@@ -263,6 +266,7 @@ ynh_backup_if_checksum_is_different() {
             fi
         fi
     fi
+    set -o xtrace # set -x
 }
 
 # Delete a file checksum from the app settings

helpers/helpers.v2.1.d/getopts

@@ -46,10 +46,13 @@
 #
 # Requires YunoHost version 3.2.2 or higher.
 ynh_handle_getopts_args() {
+    # Trick to only re-enable debugging if it was set before
+    local xtrace_enable=$(set +o | grep xtrace)
+
     # Manage arguments only if there's some provided
     set +o xtrace # set +x
     if [ $# -eq 0 ]; then
-        set -o xtrace # set -x
+        eval "$xtrace_enable"
         return
     fi
 
@@ -181,5 +184,5 @@ ynh_handle_getopts_args() {
     # Call parse_arg and pass the modified list of args as an array of arguments.
     parse_arg "${arguments[@]}"
     
-    set -o xtrace # set -x
+    eval "$xtrace_enable"
 }

helpers/helpers.v2.1.d/go

@@ -8,19 +8,18 @@ ynh_go_try_bash_extension() {
   fi
 }
 
-goenv_install_dir="/opt/goenv"
+readonly GOENV_INSTALL_DIR="/opt/goenv"
-go_version_path="$goenv_install_dir/versions"
 # goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable.
-export GOENV_ROOT="$goenv_install_dir"
+export GOENV_ROOT="$GOENV_INSTALL_DIR"
 
 _ynh_load_go_in_path_and_other_tweaks() {
 
     # Get the absolute path of this version of go
-    local go_path="$go_version_path/$app/bin"
+    go_dir="$GOENV_INSTALL_DIR/versions/$app/bin"
 
     # Load the path of this version of go in $PATH
-    if [[ :$PATH: != *":$go_path"* ]]; then
+    if [[ :$PATH: != *":$go_dir"* ]]; then
-        PATH="$go_path:$PATH"
+        PATH="$go_dir:$PATH"
     fi
 
     # Export PATH such that it's available through sudo -E / ynh_exec_as $app
@@ -32,7 +31,7 @@ _ynh_load_go_in_path_and_other_tweaks() {
 
     # Sets the local application-specific go version
     pushd ${install_dir}
-        $goenv_install_dir/bin/goenv local $go_version
+        $GOENV_INSTALL_DIR/bin/goenv local $go_version
     popd
 }
 
@@ -56,7 +55,7 @@ ynh_go_install () {
     [[ -n "${go_version:-}" ]] || ynh_die "\$go_version should be defined prior to calling ynh_go_install"
 
     # Load goenv path in PATH
-    local CLEAR_PATH="$goenv_install_dir/bin:$PATH"
+    local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH"
 
     # Remove /usr/local/bin in PATH in case of Go prior installation
     PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
@@ -65,9 +64,9 @@ ynh_go_install () {
     test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv
 
     # Install or update goenv
-    mkdir -p $goenv_install_dir
+    mkdir -p $GOENV_INSTALL_DIR
-    pushd "$goenv_install_dir"
+    pushd "$GOENV_INSTALL_DIR"
-        if ! [ -x "$goenv_install_dir/bin/goenv" ]; then
+        if ! [ -x "$GOENV_INSTALL_DIR/bin/goenv" ]; then
             ynh_print_info "Downloading goenv..."
             git init -q
             git remote add origin https://github.com/syndbg/goenv.git 
@@ -78,13 +77,13 @@ ynh_go_install () {
         local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
         git checkout -q "$git_latest_tag"
         ynh_go_try_bash_extension
-        goenv=$goenv_install_dir/bin/goenv
+        goenv=$GOENV_INSTALL_DIR/bin/goenv
     popd
 
     # Install or update xxenv-latest
-    mkdir -p "$goenv_install_dir/plugins/xxenv-latest"
+    mkdir -p "$GOENV_INSTALL_DIR/plugins/xxenv-latest"
-    pushd "$goenv_install_dir/plugins/xxenv-latest"
+    pushd "$GOENV_INSTALL_DIR/plugins/xxenv-latest"
-        if ! [ -x "$goenv_install_dir/plugins/xxenv-latest/bin/goenv-latest" ]; then
+        if ! [ -x "$GOENV_INSTALL_DIR/plugins/xxenv-latest/bin/goenv-latest" ]; then
             ynh_print_info "Downloading xxenv-latest..."
             git init -q
             git remote add origin https://github.com/momo-lab/xxenv-latest.git 
@@ -97,10 +96,10 @@ ynh_go_install () {
     popd
 
     # Enable caching
-    mkdir -p "${goenv_install_dir}/cache"
+    mkdir -p "${GOENV_INSTALL_DIR}/cache"
 
     # Create shims directory if needed
-    mkdir -p "${goenv_install_dir}/shims"
+    mkdir -p "${GOENV_INSTALL_DIR}/shims"
 
     # Restore /usr/local/bin in PATH
     PATH=$CLEAR_PATH
@@ -109,7 +108,7 @@ ynh_go_install () {
     test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go
 
     # Install the requested version of Go
-    local final_go_version=$(goenv latest --print "$go_version")
+    local final_go_version=$("$goenv_latest_dir/bin/goenv-latest" --print "$go_version")
     ynh_print_info "Installation of Go-$final_go_version"
     goenv install --quiet --skip-existing "$final_go_version" 2>&1
 
@@ -122,8 +121,8 @@ ynh_go_install () {
 
     # Set environment for Go users
     echo  "#goenv
-export GOENV_ROOT=$goenv_install_dir
+export GOENV_ROOT=$GOENV_INSTALL_DIR
-export PATH=\"$goenv_install_dir/bin:$PATH\"
+export PATH=\"$GOENV_INSTALL_DIR/bin:$PATH\"
 eval \"\$(goenv init -)\"
 #goenv" > /etc/profile.d/goenv.sh
 
@@ -142,7 +141,7 @@ ynh_go_remove () {
     local go_version=$(ynh_app_setting_get --key="go_version")
 
     # Load goenv path in PATH
-    local CLEAR_PATH="$goenv_install_dir/bin:$PATH"
+    local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH"
 
     # Remove /usr/local/bin in PATH in case of Go prior installation
     PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
@@ -182,7 +181,7 @@ _ynh_go_cleanup () {
         if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1`
         then
             ynh_print_info "Removing of Go-$installed_go_version"
-            $goenv_install_dir/bin/goenv uninstall --force "$installed_go_version"
+            $GOENV_INSTALL_DIR/bin/goenv uninstall --force "$installed_go_version"
         fi
     done
 
@@ -191,7 +190,7 @@ _ynh_go_cleanup () {
     then
         # Remove goenv environment configuration
         ynh_print_info "Removing of goenv"
-        ynh_safe_rm "$goenv_install_dir"
+        ynh_safe_rm "$GOENV_INSTALL_DIR"
         ynh_safe_rm "/etc/profile.d/goenv.sh"
     fi
 }

helpers/helpers.v2.1.d/logrotate

@@ -14,7 +14,7 @@ FIRST_CALL_TO_LOGROTATE="true"
 # Requires YunoHost version 2.6.4 or higher.
 ynh_config_add_logrotate() {
 
-    logfile="$1"
+    local logfile="${1:-}"
 
     set -o noglob
     if [[ -z "$logfile" ]]; then

helpers/helpers.v2.1.d/nodejs

@@ -1,18 +1,17 @@
 #!/bin/bash
 
-n_install_dir="/opt/node_n"
+readonly N_INSTALL_DIR="/opt/node_n"
-node_version_path="$n_install_dir/n/versions/node"
 # N_PREFIX is the directory of n, it needs to be loaded as a environment variable.
-export N_PREFIX="$n_install_dir"
+export N_PREFIX="$N_INSTALL_DIR"
 
 _ynh_load_nodejs_in_path_and_other_tweaks() {
 
     # Get the absolute path of this version of node
-    local nodejs_path="$node_version_path/$nodejs_version/bin"
+    nodejs_dir="$N_INSTALL_DIR/n/versions/node/$nodejs_version/bin"
 
     # Load the path of this version of node in $PATH
-    if [[ :$PATH: != *":$nodejs_path"* ]]; then
+    if [[ :$PATH: != *":$nodejs_dir"* ]]; then
-        PATH="$nodejs_path:$PATH"
+        PATH="$nodejs_dir:$PATH"
     fi
 
     # Export PATH such that it's available through sudo -E / ynh_exec_as $app
@@ -46,11 +45,11 @@ ynh_nodejs_install() {
 
     [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_install"
 
-    # Create $n_install_dir
+    # Create $N_INSTALL_DIR
-    mkdir --parents "$n_install_dir"
+    mkdir --parents "$N_INSTALL_DIR"
 
     # Load n path in PATH
-    CLEAR_PATH="$n_install_dir/bin:$PATH"
+    CLEAR_PATH="$N_INSTALL_DIR/bin:$PATH"
     # Remove /usr/local/bin in PATH in case of node prior installation
     PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
 
@@ -59,10 +58,10 @@ ynh_nodejs_install() {
     test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n
 
     # Install (or update if YunoHost vendor/ folder updated since last install) n
-    mkdir -p $n_install_dir/bin/
+    mkdir -p $N_INSTALL_DIR/bin/
-    cp "$YNH_HELPERS_DIR/vendor/n/n" $n_install_dir/bin/n
+    cp "$YNH_HELPERS_DIR/vendor/n/n" $N_INSTALL_DIR/bin/n
     # Tweak for n to understand it's installed in $N_PREFIX
-    ynh_replace --match="^N_PREFIX=\${N_PREFIX-.*}$" --replace="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --file="$n_install_dir/bin/n"
+    ynh_replace --match="^N_PREFIX=\${N_PREFIX-.*}$" --replace="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --file="$N_INSTALL_DIR/bin/n"
 
     # Restore /usr/local/bin in PATH
     PATH=$CLEAR_PATH
@@ -80,16 +79,18 @@ ynh_nodejs_install() {
     fi
 
     # Find the last "real" version for this major version of node.
-    real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1)
+    real_nodejs_version=$(find $N_INSTALL_DIR/n/versions/node/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1)
     real_nodejs_version=$(basename $real_nodejs_version)
 
     # Create a symbolic link for this major version if the file doesn't already exist
-    if [ ! -e "$node_version_path/$nodejs_version" ]; then
+    if [ ! -e "$N_INSTALL_DIR/n/versions/node/$nodejs_version" ]; then
-        ln --symbolic --force --no-target-directory $node_version_path/$real_nodejs_version $node_version_path/$nodejs_version
+        ln --symbolic --force --no-target-directory \
+            $N_INSTALL_DIR/n/versions/node/$real_nodejs_version \
+            $N_INSTALL_DIR/n/versions/node/$nodejs_version
     fi
 
     # Store the ID of this app and the version of node requested for it
-    echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
+    echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$N_INSTALL_DIR/ynh_app_version"
 
     # Store nodejs_version into the config of this app
     ynh_app_setting_set --key=nodejs_version --value=$nodejs_version
@@ -111,17 +112,16 @@ ynh_nodejs_remove() {
     [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_remove"
 
     # Remove the line for this app
-    sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version"
+    sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$N_INSTALL_DIR/ynh_app_version"
 
     # If no other app uses this version of nodejs, remove it.
-    if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"; then
+    if ! grep --quiet "$nodejs_version" "$N_INSTALL_DIR/ynh_app_version"; then
-        $n_install_dir/bin/n rm $nodejs_version
+        $N_INSTALL_DIR/bin/n rm $nodejs_version
     fi
 
     # If no other app uses n, remove n
-    if [ ! -s "$n_install_dir/ynh_app_version" ]; then
+    if [ ! -s "$N_INSTALL_DIR/ynh_app_version" ]; then
-        ynh_safe_rm "$n_install_dir"
+        ynh_safe_rm "$N_INSTALL_DIR"
-        ynh_safe_rm "/usr/local/n"
         sed --in-place "/N_PREFIX/d" /root/.bashrc
     fi
 }

helpers/helpers.v2.1.d/php

@@ -19,89 +19,68 @@ fi
 #
 # usage: ynh_config_add_phpfpm
 #
-# This helper assumes the app has an conf/extra_php-fpm.conf snippet
+# This will automatically generate an appropriate PHP-FPM configuration for this app.
 #
-# The actual PHP configuration will be automatically generated, 
+# The resulting configuration will be deployed to the appropriate place:
-# and your extra_php-fpm.conf will be appended (typically contains PHP upload limits)
+# /etc/php/$php_version/fpm/pool.d/$app.conf
 #
-# The resulting configuration will be deployed to the appropriate place, /etc/php/$php_version/fpm/pool.d/$app.conf
+# If the app provides a conf/extra_php-fpm.conf template, it will be appended
+# to the generated configuration. (In the vast majority of cases, this shouldnt
+# be necessary)
 #
-# Performance-related options in the PHP conf, such as : 
+# $php_version should be defined prior to calling this helper, but there should
-# pm.max_children, pm.start_servers, pm.min_spare_servers pm.max_spare_servers
+# be no reason to manually set it, as it is automatically set by the apt
-# are computed from two parameters called "usage" and "footprint" which can be set to low/medium/high. (cf details below)
+# helpers/resources when installing phpX.Y dependencies (PHP apps should at
+# least install phpX.Y-fpm using the apt helper/resource)
 #
-# If you wish to tweak those, please initialize the settings `fpm_usage` and `fpm_footprint` 
+# $php_group can be defined as a global (from _common.sh) if the worker
-# *prior* to calling this helper. Otherwise, "low" will be used as a default for both values.
+# processes should run with a different group than $app
 #
-# Otherwise, if you want the user to have control over these, we encourage to create a config panel 
+# Additional "pm" and "php_admin_value" settings which are meant to be possibly
-# (which should ultimately be standardized by the core ...)
+# configurable by admins from a future standard config panel at some point,
+# related to performance and availability of the app, for which tweaking may be
+# required if the app is used by "plenty" of users and other memory/CPU load
+# considerations....
 #
-# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM.
+# If you have good reasons to be willing to use different
-# So it will be used to defined 'pm.max_children'
+# defaults than the one set by this helper (while still allowing admin to
-# A lower value for the footprint will allow more children for 'pm.max_children'. And so for
+# override it) you should use `ynh_app_setting_set_default`
-#    'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the
-#    value of 'pm.max_children'
-# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores.
 #
-# The usage value will defined the way php will handle the children for the pool.
+# - $php_upload_max_filezise: corresponds upload_max_filesize and post_max_size. Defaults to 50M
-# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the
+# - $php_process_management: corresponds to "pm" (ondemand, dynamic, static). Defaults to ondemand
-#   service is used, otherwise no child will stay alive. This config gives the lower footprint when the
+# - $php_max_children: by default, computed from "total RAM" divided by 40, cf _default_php_max_children
-#   service is idle. But will use more proc since it has to start a child as soon it's used.
+# - $php_memory_limit: by default, 128M (from global php.ini)
-# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children
+#
-#   equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request.
+# Note that if $php_process_management is set to "dynamic", then these
-#   The number of children can grow if needed. The footprint can stay low if the service is idle, but
+# variables MUST be defined prior to calling the helper (no default value) ...
-#   not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few
+# Check PHP-FPM's manual for more info on what these are (: ...
-#   children already available.
+#
-# Set as 'high', the process manager will be set at 'static'. There will be always as many children as
+# - $php_start_servers
-#   'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum
+# - $php_min_spare_servers
-#   RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many
+# - $php_max_spare_servers
-#   children ready to answer.
 #
-# Requires YunoHost version 4.1.0 or higher.
 ynh_config_add_phpfpm() {
-    # ============ Argument parsing =============
-    local -A args_array=([g]=group=)
-    local group
-    ynh_handle_getopts_args "$@"
-    group=${group:-}
-    # ===========================================
 
-    # If the PHP version changed, remove the old fpm conf
+    [[ -n "${php_version:-}" ]] || ynh_die "\$php_version should be defined prior to calling ynh_config_add_phpfpm. You should not need to define it manually, it is automatically set by the apt helper when installing the phpX.Y- depenencies"
-    # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper)
-    # FIXME: so is this still needed @_@
-    local old_php_version=$(ynh_app_setting_get --key=php_version)
-    if [ -n "$old_php_version" ] && [ "$old_php_version" != "$php_version" ]; then
-        local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir)
-        local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
 
-        if [[ -f "$old_php_finalphpconf" ]]
+    # Apps may define $php_group as a global (e.g. from _common.sh) to change this
-        then
+    # (this is not meant to be overridable by users)
-            ynh_backup_if_checksum_is_different "$old_php_finalphpconf"
+    local php_group=${php_group:-$app}
-            ynh_remove_fpm_config
-        fi
-    fi
 
-    local fpm_service="php${php_version}-fpm"
+    # Meant to be overridable by users from a standard config panel at some point ...
-    local fpm_config_dir="/etc/php/$php_version/fpm"
+    # Apps willing to tweak these should use ynh_setting_set_default_value (in install and upgrade?)
+    #
+    local php_upload_max_filesize=${php_upload_max_filesize:-50M}
+    local php_process_management=${php_process_management:-ondemand}  # alternatively 'dynamic' or 'static'
+    local php_max_children=${php_max_children:-$(_default_php_max_children)}
+    local php_memory_limit=${php_memory_limit:-128M}  # default value is from global php.ini
 
-    # Create the directory for FPM pools
+    local phpfpm_template=$(mktemp)
-    mkdir --parents "$fpm_config_dir/pool.d"
+    cat << EOF > $phpfpm_template
-
-    # FIXME: zzzz do we really need those ...
-    ynh_app_setting_set --key=fpm_config_dir --value="$fpm_config_dir"
-    ynh_app_setting_set --key=fpm_service --value="$fpm_service"
-    ynh_app_setting_set --key=php_version --value=$php_version
-
-    # Define the values to use for the configuration of PHP.
-    _ynh_get_scalable_phpfpm
-
-    local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app")
-    local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf"
-    echo "
 [__APP__]
 
 user = __APP__
-group = __PHPFPM_GROUP__
+group = __PHP_GROUP__
 
 chdir = __INSTALL_DIR__
 
@@ -109,39 +88,48 @@ listen = /var/run/php/php__PHP_VERSION__-fpm-__APP__.sock
 listen.owner = www-data
 listen.group = www-data
 
-pm = __PHP_PM__
+pm = __PHP_PROCESS_MANAGEMENT__
 pm.max_children = __PHP_MAX_CHILDREN__
 pm.max_requests = 500
 request_terminate_timeout = 1d
-" >"$phpfpm_path"
 
-    if [ "$php_pm" = "dynamic" ]; then
+EOF
-        echo "
+    if [ "$php_process_management" = "dynamic" ]; then
+        cat << EOF >> $phpfpm_template
 pm.start_servers = __PHP_START_SERVERS__
 pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__
 pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__
-" >>"$phpfpm_path"
+EOF
-
+    elif [ "$php_process_management" = "ondemand" ]; then
-    elif [ "$php_pm" = "ondemand" ]; then
+        cat << EOF >> $phpfpm_template
-        echo "
 pm.process_idle_timeout = 10s
-" >>"$phpfpm_path"
+EOF
     fi
 
-    # Concatene the extra config.
+    cat << EOF >> $phpfpm_template
+php_admin_value[upload_max_filesize] = __PHP_UPLOAD_MAX_FILESIZE__
+php_admin_value[post_max_size] = __PHP_UPLOAD_MAX_FILESIZE__
+php_admin_value[memory_limit] = __PHP_MEMORY_LIMIT__
+EOF
+
+    # Concatene the extra config
     if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then
-        cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path"
+        cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_template"
     fi
 
-    ynh_config_add --template="$phpfpm_path" --destination="$fpm_config_dir/pool.d/$app.conf"
+    # Make sure the fpm pool dir exists
+    mkdir --parents "/etc/php/$php_version/fpm/pool.d"
+    # And hydrate configuration
+    ynh_config_add --template="$phpfpm_template" --destination="/etc/php/$php_version/fpm/pool.d/$app.conf"
 
     # Validate that the new php conf doesn't break php-fpm entirely
     if ! php-fpm${php_version} --test 2>/dev/null; then
         php-fpm${php_version} --test || true
-        ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf"
+        ynh_safe_rm "/etc/php/$php_version/fpm/pool.d/$app.conf"
         ynh_die "The new configuration broke php-fpm?"
     fi
-    ynh_systemctl --service=$fpm_service --action=reload
+
+    ynh_systemctl --service=php${php_version}-fpm --action=reload
 }
 
 # Remove the dedicated PHP-FPM config
@@ -150,125 +138,31 @@ pm.process_idle_timeout = 10s
 #
 # Requires YunoHost version 2.7.2 or higher.
 ynh_config_remove_phpfpm() {
-    local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir)
+    ynh_safe_rm "/etc/php/$php_version/fpm/pool.d/$app.conf"
-
-    ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf"
     ynh_systemctl --service="php${php_version}-fpm" --action=reload
 }
 
-# Define the values to configure PHP-FPM
+_default_php_max_children() {
-#
+    # Get the total of RAM available
-# [internal]
+    local total_ram=$(ynh_get_ram --total)
-#
-# usage: _ynh_get_scalable_phpfpm
-# Footprint can be defined via the "fpm_footprint", to be set prior to calling this helper
-# low    - Less than 20 MB of RAM by pool.
-# medium - Between 20 MB and 40 MB of RAM by pool.
-# high   - More than 40 MB of RAM by pool.
-# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value.
-# To have this value, use the following command and stress the service.
-# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP
-#
-# Usage can be defined via the "fpm_usage", to be set prior to calling this helper
-# low    - Personal usage, behind the SSO.
-# medium - Low usage, few people or/and publicly accessible.
-# high   - High usage, frequently visited website.
-#
-_ynh_get_scalable_phpfpm() {
-
-    # If no usage provided, default to the value existing in setting ... or to low
-    local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage)
-    local usage=${fpm_usage_in_setting:-low}
-    ynh_app_setting_set --key=fpm_usage --value=$usage
-
-    # If no footprint provided, default to the value existing in setting ... or to low
-    local fpm_footprint_in_setting=$(ynh_app_setting_get --key=fpm_footprint)
-    local footprint=${fpm_footprint_in_setting:-low}
-    ynh_app_setting_set --key=fpm_footprint --value=$footprint
-
-    # Set all characters as lowercase
-    if [ "$footprint" = "low" ]; then
-        footprint=20
-    elif [ "$footprint" = "medium" ]; then
-        footprint=35
-    elif [ "$footprint" = "high" ]; then
-        footprint=50
-    fi
-
-    # Define the factor to determine min_spare_servers
-    # to avoid having too few children ready to start for heavy apps
-    if [ $footprint -le 20 ]; then
-        min_spare_servers_factor=8
-    elif [ $footprint -le 35 ]; then
-        min_spare_servers_factor=5
-    else
-        min_spare_servers_factor=3
-    fi
-
-    # Define the way the process manager handle child processes.
-    if [ "$usage" = "low" ]; then
-        php_pm=ondemand
-    elif [ "$usage" = "medium" ]; then
-        php_pm=dynamic
-    elif [ "$usage" = "high" ]; then
-        php_pm=static
-    else
-        ynh_die "Does not recognize '$usage' as an usage value."
-    fi
-
-    # Get the total of RAM available, except swap.
-    local max_ram=$(ynh_get_ram --total)
-
-    at_least_one() {
-        # Do not allow value below 1
-        if [ $1 -le 0 ]; then
-            echo 1
-        else
-            echo $1
-        fi
-    }
-
-    # Define pm.max_children
-    # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app.
-    # So if PHP-FPM start the maximum of children, it won't exceed half of the ram.
-    php_max_children=$(($max_ram / 2 / $footprint))
-    # If process manager is set as static, use half less children.
-    # Used as static, there's always as many children as the value of pm.max_children
-    if [ "$php_pm" = "static" ]; then
-        php_max_children=$(($php_max_children / 2))
-    fi
-    php_max_children=$(at_least_one $php_max_children)
 
+    # The value of pm.max_children is the total amount of ram divide by 2,
+    # divide again by 20MB (= a default, classic worker footprint) This is
+    # designed such that if PHP-FPM start the maximum of children, it won't
+    # exceed half of the ram.
+    local php_max_children="$(($total_ram / 40))"
+    # Make sure we get at least max_children = 1
+    if [ $php_max_children -le 0 ]; then
+        php_max_children=1
     # To not overload the proc, limit the number of children to 4 times the number of cores.
-    local core_number=$(nproc)
+    elif [ $php_max_children -gt "$(($(nproc) * 4))" ]; then
-    local max_proc=$(($core_number * 4))
+        php_max_children="$(($(nproc) * 4))"
-    if [ $php_max_children -gt $max_proc ]; then
-        php_max_children=$max_proc
     fi
 
-    # Get a potential forced value for php_max_children
+    echo "$php_max_children"
-    local php_forced_max_children=$(ynh_app_setting_get --key=php_forced_max_children)
-    if [ -n "$php_forced_max_children" ]; then
-        php_max_children=$php_forced_max_children
-    fi
-
-    if [ "$php_pm" = "dynamic" ]; then
-        # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager
-        php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor))
-        php_min_spare_servers=$(at_least_one $php_min_spare_servers)
-
-        php_max_spare_servers=$(($php_max_children / 2))
-        php_max_spare_servers=$(at_least_one $php_max_spare_servers)
-
-        php_start_servers=$(($php_min_spare_servers + ($php_max_spare_servers - $php_min_spare_servers) / 2))
-        php_start_servers=$(at_least_one $php_start_servers)
-    else
-        php_min_spare_servers=0
-        php_max_spare_servers=0
-        php_start_servers=0
-    fi
 }
 
+
 # Execute a command with Composer
 #
 # Will use $install_dir as workdir unless $composer_workdir exists (but that shouldnt be necessary)

helpers/helpers.v2.1.d/postgresql

@@ -81,6 +81,8 @@ ynh_psql_dump_db() {
 # | arg: pwd - the password to identify user by
 #
 ynh_psql_create_user() {
+    local user=$1
+    local pwd=$2
     sudo --login --user=postgres psql <<< "CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'"
 }
 

helpers/helpers.v2.1.d/ruby

@@ -1,20 +1,19 @@
 #!/bin/bash
 
-rbenv_install_dir="/opt/rbenv"
+readonly RBENV_INSTALL_DIR="/opt/rbenv"
-ruby_version_path="$rbenv_install_dir/versions"
 
 # RBENV_ROOT is the directory of rbenv, it needs to be loaded as a environment variable.
-export RBENV_ROOT="$rbenv_install_dir"
+export RBENV_ROOT="$RBENV_INSTALL_DIR"
-export rbenv_root="$rbenv_install_dir"
+export rbenv_root="$RBENV_INSTALL_DIR"
 
 _ynh_load_ruby_in_path_and_other_tweaks() {
 
     # Get the absolute path of this version of Ruby
-    local ruby_path="$ruby_version_path/$app/bin"
+    ruby_dir="$RBENV_INSTALL_DIR/versions/$app/bin"
 
     # Load the path of this version of ruby in $PATH
-    if [[ :$PATH: != *":$ruby_path"* ]]; then
+    if [[ :$PATH: != *":$ruby_dir"* ]]; then
-        PATH="$ruby_path:$PATH"
+        PATH="$ruby_dir:$PATH"
     fi
 
     # Export PATH such that it's available through sudo -E / ynh_exec_as $app
@@ -26,7 +25,7 @@ _ynh_load_ruby_in_path_and_other_tweaks() {
 
     # Sets the local application-specific Ruby version
     pushd ${install_dir}
-        $rbenv_install_dir/bin/rbenv local $ruby_version
+        $RBENV_INSTALL_DIR/bin/rbenv local $ruby_version
     popd
 }
 
@@ -55,7 +54,7 @@ ynh_ruby_install () {
     [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_install"
 
     # Load rbenv path in PATH
-    local CLEAR_PATH="$rbenv_install_dir/bin:$PATH"
+    local CLEAR_PATH="$RBENV_INSTALL_DIR/bin:$PATH"
 
     # Remove /usr/local/bin in PATH in case of Ruby prior installation
     PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
@@ -64,8 +63,8 @@ ynh_ruby_install () {
     test -x /usr/bin/ruby && mv /usr/bin/ruby /usr/bin/ruby_rbenv
 
     # Install or update rbenv
-    mkdir -p $rbenv_install_dir
+    mkdir -p $RBENV_INSTALL_DIR
-    rbenv="$(command -v rbenv $rbenv_install_dir/bin/rbenv | grep "$rbenv_install_dir/bin/rbenv" | head -1)"
+    rbenv="$(command -v rbenv $RBENV_INSTALL_DIR/bin/rbenv | grep "$RBENV_INSTALL_DIR/bin/rbenv" | head -1)"
     if [ -n "$rbenv" ]; then
         pushd "${rbenv%/*/*}"
             if git remote -v 2>/dev/null | grep "https://github.com/rbenv/rbenv.git"; then
@@ -75,30 +74,30 @@ ynh_ruby_install () {
             else
                 echo "Reinstalling rbenv..."
                 cd ..
-                ynh_safe_rm $rbenv_install_dir
+                ynh_safe_rm $RBENV_INSTALL_DIR
-                mkdir -p $rbenv_install_dir
+                mkdir -p $RBENV_INSTALL_DIR
-                cd $rbenv_install_dir
+                cd $RBENV_INSTALL_DIR
                 git init -q
                 git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1
                 git checkout -q -b master origin/master
                 ynh_ruby_try_bash_extension
-                rbenv=$rbenv_install_dir/bin/rbenv
+                rbenv=$RBENV_INSTALL_DIR/bin/rbenv
             fi
         popd
     else
         echo "Installing rbenv..."
-        pushd $rbenv_install_dir
+        pushd $RBENV_INSTALL_DIR
             git init -q
             git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1
             git checkout -q -b master origin/master
             ynh_ruby_try_bash_extension
-            rbenv=$rbenv_install_dir/bin/rbenv
+            rbenv=$RBENV_INSTALL_DIR/bin/rbenv
         popd
     fi
 
-    mkdir -p "${rbenv_install_dir}/plugins"
+    mkdir -p "${RBENV_INSTALL_DIR}/plugins"
 
-    ruby_build="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-install rbenv-install | head -1)"
+    ruby_build="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-install rbenv-install | head -1)"
     if [ -n "$ruby_build" ]; then
         pushd "${ruby_build%/*/*}"
             if git remote -v 2>/dev/null | grep "https://github.com/rbenv/ruby-build.git"; then
@@ -108,10 +107,10 @@ ynh_ruby_install () {
         popd
     else
         echo "Installing ruby-build..."
-        git clone -q https://github.com/rbenv/ruby-build.git "${rbenv_install_dir}/plugins/ruby-build"
+        git clone -q https://github.com/rbenv/ruby-build.git "${RBENV_INSTALL_DIR}/plugins/ruby-build"
     fi
 
-    rbenv_alias="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)"
+    rbenv_alias="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)"
     if [ -n "$rbenv_alias" ]; then
         pushd "${rbenv_alias%/*/*}"
             if git remote -v 2>/dev/null | grep "https://github.com/tpope/rbenv-aliases.git"; then
@@ -121,10 +120,10 @@ ynh_ruby_install () {
         popd
     else
         echo "Installing rbenv-aliases..."
-        git clone -q https://github.com/tpope/rbenv-aliases.git "${rbenv_install_dir}/plugins/rbenv-aliase"
+        git clone -q https://github.com/tpope/rbenv-aliases.git "${RBENV_INSTALL_DIR}/plugins/rbenv-aliase"
     fi
 
-    rbenv_latest="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)"
+    rbenv_latest="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)"
     if [ -n "$rbenv_latest" ]; then
         pushd "${rbenv_latest%/*/*}"
             if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then
@@ -134,14 +133,14 @@ ynh_ruby_install () {
         popd
     else
         echo "Installing xxenv-latest..."
-        git clone -q https://github.com/momo-lab/xxenv-latest.git "${rbenv_install_dir}/plugins/xxenv-latest"
+        git clone -q https://github.com/momo-lab/xxenv-latest.git "${RBENV_INSTALL_DIR}/plugins/xxenv-latest"
     fi
 
     # Enable caching
-    mkdir -p "${rbenv_install_dir}/cache"
+    mkdir -p "${RBENV_INSTALL_DIR}/cache"
 
     # Create shims directory if needed
-    mkdir -p "${rbenv_install_dir}/shims"
+    mkdir -p "${RBENV_INSTALL_DIR}/shims"
 
     # Restore /usr/local/bin in PATH
     PATH=$CLEAR_PATH
@@ -175,8 +174,8 @@ ynh_ruby_install () {
 
     # Set environment for Ruby users
     echo  "#rbenv
-export RBENV_ROOT=$rbenv_install_dir
+export RBENV_ROOT=$RBENV_INSTALL_DIR
-export PATH=\"$rbenv_install_dir/bin:$PATH\"
+export PATH=\"$RBENV_INSTALL_DIR/bin:$PATH\"
 eval \"\$(rbenv init -)\"
 #rbenv" > /etc/profile.d/rbenv.sh
 
@@ -192,10 +191,11 @@ eval \"\$(rbenv init -)\"
 #
 # usage: ynh_ruby_remove
 ynh_ruby_remove () {
-    local ruby_version=$(ynh_app_setting_get --key=ruby_version)
+
+    [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_remove"
 
     # Load rbenv path in PATH
-    local CLEAR_PATH="$rbenv_install_dir/bin:$PATH"
+    local CLEAR_PATH="$RBENV_INSTALL_DIR/bin:$PATH"
 
     # Remove /usr/local/bin in PATH in case of Ruby prior installation
     PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
@@ -235,7 +235,7 @@ _ynh_ruby_cleanup () {
         if ! echo ${required_ruby_versions} | grep -q "${installed_ruby_version}"
         then
             echo "Removing Ruby-$installed_ruby_version"
-            $rbenv_install_dir/bin/rbenv uninstall --force $installed_ruby_version
+            $RBENV_INSTALL_DIR/bin/rbenv uninstall --force $installed_ruby_version
         fi
     done
 
@@ -244,7 +244,7 @@ _ynh_ruby_cleanup () {
     then
         # Remove rbenv environment configuration
         echo "Removing rbenv"
-        ynh_safe_rm "$rbenv_install_dir"
+        ynh_safe_rm "$RBENV_INSTALL_DIR"
         ynh_safe_rm "/etc/profile.d/rbenv.sh"
     fi
 }

helpers/helpers.v2.1.d/setting

@@ -42,6 +42,41 @@ ynh_app_setting_set() {
     ynh_app_setting "set" "$app" "$key" "$value"
 }
 
+# Set an application setting but only if the "$key" variable ain't set yet
+#
+# Note that it doesn't just define the setting but ALSO define the $foobar variable
+#
+# Hence it's meant as a replacement for this legacy overly complex syntax:
+#
+# if [ -z "${foo:-}" ]
+# then
+#     foo="bar"
+#     ynh_app_setting_set --key="foo" --value="$foo"
+# fi
+#
+# usage: ynh_app_setting_set_default --app=app --key=key --value=value
+# | arg: -a, --app=     - the application id
+# | arg: -k, --key=     - the setting name to set
+# | arg: -v, --value=   - the default setting value to set
+#
+# Requires YunoHost version 11.1.16 or higher.
+ynh_app_setting_set_default() {
+    # ============ Argument parsing =============
+    local _globalapp=${app-:}
+    local -A args_array=([a]=app= [k]=key= [v]=value=)
+    local app
+    local key
+    local value
+    ynh_handle_getopts_args "$@"
+    app="${app:-$_globalapp}"
+    # ===========================================
+
+    if [ -z "${!key:-}" ]; then
+        eval $key=\$value
+        ynh_app_setting "set" "$app" "$key" "$value"
+    fi
+}
+
 # Delete an application setting
 #
 # usage: ynh_app_setting_delete --app=app --key=key
@@ -68,6 +103,8 @@ ynh_app_setting_delete() {
 # [internal]
 #
 ynh_app_setting() {
+    # Trick to only re-enable debugging if it was set before
+    local xtrace_enable=$(set +o | grep xtrace)
     set +o xtrace # set +x
     ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <<EOF
 import os, yaml, sys
@@ -91,5 +128,5 @@ else:
     with open(setting_file, "w") as f:
         yaml.safe_dump(settings, f, default_flow_style=False)
 EOF
-    set -o xtrace # set -x
+    eval "$xtrace_enable"
 }

helpers/helpers.v2.1.d/systemd

@@ -49,7 +49,7 @@ ynh_config_remove_systemd() {
 # | arg: -a, --action=       - Action to perform with systemctl. Default: start
 # | arg: -w, --wait_until=   - The pattern to find in the log to attest the service is effectively fully started.
 # | arg: -p, --log_path=     - Log file - Path to the log file. Default : `/var/log/$app/$app.log`
-# | arg: -t, --timeout=      - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds.
+# | arg: -t, --timeout=      - Timeout - The maximum time to wait before ending the watching. Default : 60 seconds.
 # | arg: -e, --length=       - Length of the error log displayed for debugging : Default : 20
 #
 # Requires YunoHost version 3.5.0 or higher.
@@ -68,9 +68,15 @@ ynh_systemctl() {
     wait_until=${wait_until:-}
     length=${length:-20}
     log_path="${log_path:-/var/log/$service/$service.log}"
-    timeout=${timeout:-300}
+    timeout=${timeout:-60}
     # ===========================================
 
+    # On CI, use length=100 because it's sometime hell to debug otherwise for super-long output
+    if ynh_in_ci_tests && [ $length -le 20 ]
+    then
+        length=100
+    fi
+
     # Manage case of service already stopped
     if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service; then
         return 0
@@ -114,7 +120,7 @@ ynh_systemctl() {
 
     # Start the timeout and try to find wait_until
     if [[ -n "${wait_until:-}" ]]; then
-        set +x
+        set +o xtrace # set +x
         local i=0
         local starttime=$(date +%s)
         for i in $(seq 1 $timeout); do
@@ -144,7 +150,7 @@ ynh_systemctl() {
             fi
             sleep 1
         done
-        set -x
+        set -o xtrace # set -x
         if [ $i -ge 3 ]; then
             echo "" >&2
         fi
@@ -156,6 +162,13 @@ ynh_systemctl() {
                 ynh_print_warn "==="
                 tail --lines=$length "$log_path" >&2
             fi
+
+            # If we tried to reload/start/restart the service but systemctl consider it to be still inactive/broken, then handle it as a failure
+            if ([ "$action" == "reload" ] || [ "$action" == "start" ] || [ "$action" == "restart" ]) && ! systemctl --quiet is-active $service
+            then
+                _ynh_clean_check_starting
+                return 1
+            fi
         fi
         _ynh_clean_check_starting
     fi

helpers/helpers.v2.1.d/user

@@ -174,9 +174,9 @@ ynh_system_user_delete() {
 
 # Execute a command after sudoing as $app
 #
-# Note that exported bash env variables are kept (using -E option of sudo)
+# Note that the $PATH variable is preserved (using --preserve-env=PATH)
 #
 # usage: ynh_exec_as_app COMMAND [ARG ...]
 ynh_exec_as_app() {
-    sudo -E -u"$app" "$@"
+    sudo --preserve-env=PATH -u "$app" "$@"
 }

helpers/helpers.v2.1.d/utils

@@ -158,7 +158,6 @@ ynh_setup_source() {
 
     local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')"
     local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')"
-    local src_sumprg="sha256sum"
     local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')"
     local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')"
     src_in_subdir=${src_in_subdir:-true}
@@ -198,9 +197,6 @@ ynh_setup_source() {
         ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter"
     fi
 
-    # (Unused?) mecanism where one can have the file in a special local cache to not have to download it...
-    local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}"
-
     # Gotta use this trick with 'dirname' because source_id may contain slashes x_x
     mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id})
     src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}"
@@ -208,14 +204,10 @@ ynh_setup_source() {
     if [ "$src_format" = "docker" ]; then
         src_platform="${src_platform:-"linux/$YNH_ARCH"}"
     else
-        if test -e "$local_src"; then
-            cp $local_src $src_filename
-        fi
-
         [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?"
 
         # If the file was prefetched but somehow doesn't match the sum, rm and redownload it
-        if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
+        if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status
         then
             rm -f "$src_filename"
         fi
@@ -233,9 +225,9 @@ ynh_setup_source() {
         fi
 
         # Check the control sum
-        if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
+        if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status
         then
-            local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)"
+            local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)"
             local actual_size="$(du -hs ${src_filename} | cut --fields=1)"
             rm -f ${src_filename}
             ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})."
@@ -468,7 +460,7 @@ ynh_read_manifest() {
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_app_upstream_version() {
-    echo "${$YNH_APP_MANIFEST_VERSION/~ynh*/}"
+    echo "${YNH_APP_MANIFEST_VERSION/~ynh*/}"
 }
 
 # Return 0 if the "upstream" part of the version changed, or 1 otherwise (ie only the ~ynh suffix changed)

locales/eu.json

@@ -601,8 +601,8 @@
     "restore_hook_unavailable": "'{part}'-(e)rako lehengoratze agindua ez dago erabilgarri ez sisteman ezta fitxategian ere",
     "restore_cleaning_failed": "Ezin izan dira lehengoratzeko behin-behineko fitxategiak ezabatu",
     "restore_confirm_yunohost_installed": "Ziur al zaude dagoeneko instalatuta dagoen sistema lehengoratu nahi duzula? [{answers}]",
-    "restore_may_be_not_enough_disk_space": "Badirudi zure sistemak ez duela nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasuneko tartea: {margin} B)",
+    "restore_may_be_not_enough_disk_space": "Badirudi zure sistemak ez duela nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasun-tartea: {margin} B)",
-    "restore_not_enough_disk_space": "Ez dago nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasuneko tartea: {margin} B)",
+    "restore_not_enough_disk_space": "Ez dago nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasun-tartea: {margin} B)",
     "restore_running_hooks": "Lehengoratzeko 'hook'ak exekutatzen…",
     "restore_system_part_failed": "Ezinezkoa izan da sistemaren '{part}' atala lehengoratzea",
     "server_reboot": "Zerbitzaria berrabiaraziko da",

locales/sk.json

@@ -265,5 +265,18 @@
     "service_description_rspamd": "Filtruje spam a iné funkcie týkajúce sa e-mailu",
     "log_letsencrypt_cert_renew": "Obnoviť '{}' certifikát Let's Encrypt",
     "domain_config_cert_summary_selfsigned": "UPOZORNENIE: Aktuálny certifikát je vlastnoručne podpísaný. Prehliadače budú návštevníkom zobrazovať strašidelné varovanie!",
-    "global_settings_setting_ssowat_panel_overlay_enabled": "Povoliť malú štvorcovú ikonu portálu „YunoHost“ na aplikáciach"
+    "global_settings_setting_ssowat_panel_overlay_enabled": "Povoliť malú štvorcovú ikonu portálu „YunoHost“ na aplikáciach",
+    "domain_config_mail_out": "Odchádzajúce e-maily",
+    "domain_config_default_app": "Predvolená aplikácia",
+    "domain_config_xmpp_help": "Pozor: niektoré funkcie XMPP vyžadujú aktualizáciu vašich DNS záznamov a obnovenie Lets Encrypt certifikátu pred tým, ako je ich možné zapnúť",
+    "domain_config_default_app_help": "Návštevníci budú pri návšteve tejto domény automaticky presmerovaní na túto doménu. Ak nenastavíte žiadnu aplikáciu, zobrazí sa stránka s prihlasovacím formulárom na portál.",
+    "registrar_infos": "Informácie o registrátorovi",
+    "domain_dns_registrar_managed_in_parent_domain": "Táto doména je subdoména {parent_domain_link}. Nastavenie DNS registrátora je spravovaná v konfiguračnom paneli {parent_domain}.",
+    "log_letsencrypt_cert_install": "Inštalovať certifikát Let's Encrypt na doménu '{}'",
+    "domain_config_cert_no_checks": "Ignorovať kontroly diagnostiky",
+    "domain_config_cert_install": "Nainštalovať certifikát Let's Encrypt",
+    "domain_config_mail_in": "Prichádzajúce e-maily",
+    "domain_config_cert_summary": "Stav certifikátu",
+    "domain_config_xmpp": "Krátke správy (XMPP)",
+    "log_app_makedefault": "Nastaviť '{}' ako predvolenú aplikáciu"
 }

src/app.py

@@ -2992,19 +2992,43 @@ def _make_environment_for_app_script(
     # If packaging format v2, load all settings
     if manifest["packaging_format"] >= 2 or force_include_app_settings:
         env_dict["app"] = app
+        data_to_redact = []
+        prefixes_or_suffixes_to_redact = [
+            "pwd",
+            "pass",
+            "passwd",
+            "password",
+            "passphrase",
+            "secret",
+            "key",
+            "token",
+        ]
+
         for setting_name, setting_value in _get_app_settings(app).items():
             # Ignore special internal settings like checksum__
             # (not a huge deal to load them but idk...)
             if setting_name.startswith("checksum__"):
                 continue
 
-            env_dict[setting_name] = str(setting_value)
+            setting_value = str(setting_value)
+            env_dict[setting_name] = setting_value
+
+            # Check if we should redact this setting value
+            # (the check on the setting length exists to prevent stupid stuff like redacting empty string or something which is actually just 0/1, true/false, ...
+            if len(setting_value) > 6 and any(
+                setting_name.startswith(p) or setting_name.endswith(p)
+                for p in prefixes_or_suffixes_to_redact
+            ):
+                data_to_redact.append(setting_value)
 
         # Special weird case for backward compatibility...
         # 'path' was loaded into 'path_url' .....
         if "path" in env_dict:
             env_dict["path_url"] = env_dict["path"]
 
+        for operation_logger in OperationLogger._instances:
+            operation_logger.data_to_redact.extend(data_to_redact)
+
     return env_dict
 
 

src/log.py

@@ -45,12 +45,18 @@ LOG_FILE_EXT = ".log"
 BORING_LOG_LINES = [
     r"set [+-]x$",
     r"set [+-]o xtrace$",
+    r"\+ set \+o$",
+    r"\+ grep xtrace$",
+    r"local 'xtrace_enable=",
     r"set [+-]o errexit$",
     r"set [+-]o nounset$",
     r"trap '' EXIT",
     r"local \w+$",
     r"local exit_code=(1|0)$",
     r"local legacy_args=.*$",
+    r"local _globalapp=.*$",
+    r"local checksum_setting_name=.*$",
+    r"ynh_app_setting ",  # (note the trailing space to match the "low level" one called by other setting helpers)
     r"local -A args_array$",
     r"args_array=.*$",
     r"ret_code=1",
@@ -62,8 +68,17 @@ BORING_LOG_LINES = [
     r"\[?\['? -n '' '?\]\]?$",
     r"rm -rf /var/cache/yunohost/download/$",
     r"type -t ynh_clean_setup$",
+    r"DEBUG - \+ unset \S+$",
     r"DEBUG - \+ echo '",
+    r"DEBUG - \+ LC_ALL=C$",
+    r"DEBUG - \+ DEBIAN_FRONTEND=noninteractive$",
     r"DEBUG - \+ exit (1|0)$",
+    r"DEBUG - \+ app=\S+$",
+    r"DEBUG - \+\+ app=\S+$",
+    r"DEBUG - \+\+ jq -r .\S+$",
+    r"DEBUG - \+\+ sed 's/\^null\$//'$",
+    "DEBUG - \\+ sed --in-place \\$'s\\\\001",
+    "DEBUG - \\+ sed --in-place 's\u0001.*$",
 ]
 
 

src/utils/resources.py

@@ -156,29 +156,29 @@ class AppResource:
             app_upstream_version = manager.current["version"].split("~")[0]
 
         # FIXME : should use packaging.version to properly parse / compare versions >_>
-        self.helpers_version = None
+        self.helpers_version: float = 0
         if (
             manager
             and manager.wanted
             and manager.wanted.get("integration", {}).get("helpers_version")
         ):
-            self.helpers_version = manager.wanted.get("integration", {}).get(
+            self.helpers_version = float(
-                "helpers_version"
+                manager.wanted.get("integration", {}).get("helpers_version")
             )
         elif (
             manager
             and manager.current
             and manager.current.get("integration", {}).get("helpers_version")
         ):
-            self.helpers_version = manager.current.get("integration", {}).get(
+            self.helpers_version = float(
-                "helpers_version"
+                manager.current.get("integration", {}).get("helpers_version")
             )
         elif manager and manager.wanted and manager.wanted.get("packaging_format"):
-            self.helpers_version = str(manager.wanted.get("packaging_format"))
+            self.helpers_version = float(manager.wanted.get("packaging_format"))
         elif manager and manager.current and manager.current.get("packaging_format"):
-            self.helpers_version = str(manager.current.get("packaging_format"))
+            self.helpers_version = float(manager.current.get("packaging_format"))
         if not self.helpers_version:
-            self.helpers_version = "1"
+            self.helpers_version = 1.0
 
         replacements: dict[str, str] = {
             "__APP__": self.app,
@@ -1208,7 +1208,7 @@ class AptDependenciesAppResource(AppResource):
 
     def provision_or_update(self, context: Dict = {}):
 
-        if float(self.helpers_version) >= 2.1:
+        if self.helpers_version >= 2.1:
             ynh_apt_install_dependencies = "ynh_apt_install_dependencies"
             ynh_apt_install_dependencies_from_extra_repository = (
                 "ynh_apt_install_dependencies_from_extra_repository"
@@ -1234,7 +1234,7 @@ class AptDependenciesAppResource(AppResource):
         self._run_script("provision_or_update", script)
 
     def deprovision(self, context: Dict = {}):
-        if float(self.helpers_version) >= 2.1:
+        if self.helpers_version >= 2.1:
             ynh_apt_remove_dependencies = "ynh_apt_remove_dependencies"
         else:
             ynh_apt_remove_dependencies = "ynh_remove_app_dependencies"