diff --git a/debian/changelog b/debian/changelog index f5059d42c..1f0c6c57a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,24 @@ +yunohost (11.2.16) stable; urgency=low + + - apps/logs: fix some information not being redacted because of the packaging v2 flows (a25033bb) + - logs: misc ad-hoc tweaks to limit the noise in log sharing (06c8fbc8) + - helpers: (1/2/2.1) add a new ynh_app_setting_set_default to replace the unecessarily complex 'if [ -z ${foo:-} ]' trick ([#1873](http://github.com/YunoHost/yunohost/pull/1873)) + - helpers2.1: drop unused 'local source' mechanism from ynh_setup_source (dd8db188) + - helpers2.1: fix positional arg parsing in ynh_psql_create_user (e5585136) + - helpers2.1: rework the fpm usage/footprint madness ([#1874](http://github.com/YunoHost/yunohost/pull/1874)) + - helpers2.1: fix ynh_config_add_logrotate when no arg is passed (3942ea12) + - helpers2.1: sudo -u$app -> sudo -u $app (d4857834) + - helpers2.1: change default timeout of ynh_systemctl to 60s instead of 300s (262453f1) + - helpers2.1: display 100 lines instead of 20 in CI context when service fails to start (9298738d) + - helpers2.1: when using ynh_systemctl to reload/start/restart a service with a wait_until and it timesout, handle it as a failure rather than keep going (b3409729) + - helpers2.1: for some reason sudo -E doesn't preserve PATH even though it's exported, so we gotta explicitly use --preserve-env=PATH (5f6df6a8) + - helpers2.1: var rename / cosmetic etc for nodejs/ruby/go version and install directories (2b1f7426) + - i18n: Translations updated for Basque, Slovak + + Thanks to all contributors <3 ! (alexAubin, Jose Riha, xabirequejo) + + -- Alexandre Aubin Sun, 23 Jun 2024 15:30:22 +0200 + yunohost (11.2.15) stable; urgency=low - apps: new experimentals "2.1" helpers ([#1855](http://github.com/YunoHost/yunohost/pull/1855)) diff --git a/helpers/helpers.v1.d/setting b/helpers/helpers.v1.d/setting index 82a5d274e..c1946f6f8 100644 --- a/helpers/helpers.v1.d/setting +++ b/helpers/helpers.v1.d/setting @@ -52,6 +52,42 @@ ynh_app_setting_set() { fi } +# Set an application setting but only if the "$key" variable ain't set yet +# +# Note that it doesn't just define the setting but ALSO define the $foobar variable +# +# Hence it's meant as a replacement for this legacy overly complex syntax: +# +# if [ -z "${foo:-}" ] +# then +# foo="bar" +# ynh_app_setting_set --key="foo" --value="$foo" +# fi +# +# usage: ynh_app_setting_set_default --app=app --key=key --value=value +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting name to set +# | arg: -v, --value= - the default setting value to set +# +# Requires YunoHost version 11.1.16 or higher. +ynh_app_setting_set_default() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=akv + local -A args_array=([a]=app= [k]=key= [v]=value=) + local app + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [ -z "${!key:-}" ]; then + eval $key=\$value + ynh_app_setting "set" "$app" "$key" "$value" + fi +} + # Delete an application setting # # usage: ynh_app_setting_delete --app=app --key=key diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 520007e48..296e17b95 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -16,15 +16,14 @@ YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true" # # Requires YunoHost version 2.6.4 or higher. ynh_apt_install_dependencies() { - local dependencies=$@ - # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) - dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" - local dependencies=${dependencies//|/ | } + + # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) + local dependencies="$(sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g' <<< "$@" | sed 's/|/ | /')" local version=$(ynh_read_manifest "version") local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps # Handle specific versions - if [[ "$dependencies" =~ [\<=\>] ]]; then + if grep '[<=>]' <<< "$dependencies"; then # Replace version specifications by relationships syntax # https://www.debian.org/doc/debian-policy/ch-relationships.html # Sed clarification @@ -33,7 +32,7 @@ ynh_apt_install_dependencies() { # \+ matches one or more occurence of the previous characters, for >= or >>. # [^,]\+ matches all characters except ',' # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)' - dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')" + dependencies="$(sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g' <<< "$dependencies")" fi # ############################## # @@ -43,7 +42,7 @@ ynh_apt_install_dependencies() { # Check for specific php dependencies which requires sury # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni" # The (?<=php) syntax corresponds to lookbehind ;) - local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u) + local specific_php_version=$(grep -oP '(?<=php)[0-9.]+(?=-|\>|)' <<< "$dependencies" | sort -u) if [[ -n "$specific_php_version" ]] then @@ -57,12 +56,9 @@ ynh_apt_install_dependencies() { # If the PHP version changed, remove the old fpm conf if [ -n "$old_php_version" ] && [ "$old_php_version" != "$specific_php_version" ]; then - local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) - local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" - - if [[ -f "$old_php_finalphpconf" ]] + if [[ -f "/etc/php/$php_version/fpm/pool.d/$app.conf" ]] then - ynh_backup_if_checksum_is_different "$old_php_finalphpconf" + ynh_backup_if_checksum_is_different "/etc/php/$php_version/fpm/pool.d/$app.conf" ynh_remove_fpm_config fi fi @@ -128,21 +124,22 @@ EOF # NB: this is in a subshell (though not sure why exactly not just use pushd/popd...) cd "$TMPDIR" # Install the fake package without its dependencies with dpkg --force-depends - LC_ALL=C equivs-build ./control 2>&1 - LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" 2>&1 | tee ./dpkg_log + LC_ALL=C equivs-build ./control > ./equivs_log 2>&1 || { cat ./equivs_log; false; } + LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" > ./dpkg_log 2>&1 ) # Then install the missing dependencies with apt install - _ynh_apt_install --fix-broken \ - || { # If the installation failed - # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) - # Parse the list of problematic dependencies from dpkg's log ... - # (relevant lines look like: "foo-ynh-deps depends on bar; however:") - local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')" - # Fake an install of those dependencies to see the errors - # The sed command here is, Print only from 'Reading state info' to the end. - [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 - ynh_die "Unable to install dependencies" + _ynh_apt_install --fix-broken || { + # If the installation failed + # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) + # Parse the list of problematic dependencies from dpkg's log ... + # (relevant lines look like: "foo-ynh-deps depends on bar; however:") + cat $TMPDIR/dpkg_log + local problematic_dependencies="$(grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' $TMPDIR/dpkg_log | tr '\n' ' ')" + # Fake an install of those dependencies to see the errors + # The sed command here is, Print only from 'Reading state info' to the end. + [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 + ynh_die "Unable to install dependencies" } rm --recursive --force "$TMPDIR" # Remove the temp dir. diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index abb41e041..c310cc0b8 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -211,6 +211,7 @@ _ynh_file_checksum_exists() { # # usage: ynh_store_file_checksum /path/to/file ynh_store_file_checksum() { + set +o xtrace # set +x local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' @@ -231,6 +232,7 @@ ynh_store_file_checksum() { fi # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. unset backup_file_checksum + set -o xtrace # set -x } # Verify the checksum and backup the file if it's different @@ -240,6 +242,7 @@ ynh_store_file_checksum() { # This helper is primarily meant to allow to easily backup personalised/manually # modified config files. ynh_backup_if_checksum_is_different() { + set +o xtrace # set +x local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) @@ -263,6 +266,7 @@ ynh_backup_if_checksum_is_different() { fi fi fi + set -o xtrace # set -x } # Delete a file checksum from the app settings diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index a0ef13b13..381c74256 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -46,10 +46,13 @@ # # Requires YunoHost version 3.2.2 or higher. ynh_handle_getopts_args() { + # Trick to only re-enable debugging if it was set before + local xtrace_enable=$(set +o | grep xtrace) + # Manage arguments only if there's some provided set +o xtrace # set +x if [ $# -eq 0 ]; then - set -o xtrace # set -x + eval "$xtrace_enable" return fi @@ -181,5 +184,5 @@ ynh_handle_getopts_args() { # Call parse_arg and pass the modified list of args as an array of arguments. parse_arg "${arguments[@]}" - set -o xtrace # set -x + eval "$xtrace_enable" } diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 22797397e..23a423342 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -8,19 +8,18 @@ ynh_go_try_bash_extension() { fi } -goenv_install_dir="/opt/goenv" -go_version_path="$goenv_install_dir/versions" +readonly GOENV_INSTALL_DIR="/opt/goenv" # goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable. -export GOENV_ROOT="$goenv_install_dir" +export GOENV_ROOT="$GOENV_INSTALL_DIR" _ynh_load_go_in_path_and_other_tweaks() { # Get the absolute path of this version of go - local go_path="$go_version_path/$app/bin" + go_dir="$GOENV_INSTALL_DIR/versions/$app/bin" # Load the path of this version of go in $PATH - if [[ :$PATH: != *":$go_path"* ]]; then - PATH="$go_path:$PATH" + if [[ :$PATH: != *":$go_dir"* ]]; then + PATH="$go_dir:$PATH" fi # Export PATH such that it's available through sudo -E / ynh_exec_as $app @@ -32,7 +31,7 @@ _ynh_load_go_in_path_and_other_tweaks() { # Sets the local application-specific go version pushd ${install_dir} - $goenv_install_dir/bin/goenv local $go_version + $GOENV_INSTALL_DIR/bin/goenv local $go_version popd } @@ -56,7 +55,7 @@ ynh_go_install () { [[ -n "${go_version:-}" ]] || ynh_die "\$go_version should be defined prior to calling ynh_go_install" # Load goenv path in PATH - local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Go prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -65,9 +64,9 @@ ynh_go_install () { test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv # Install or update goenv - mkdir -p $goenv_install_dir - pushd "$goenv_install_dir" - if ! [ -x "$goenv_install_dir/bin/goenv" ]; then + mkdir -p $GOENV_INSTALL_DIR + pushd "$GOENV_INSTALL_DIR" + if ! [ -x "$GOENV_INSTALL_DIR/bin/goenv" ]; then ynh_print_info "Downloading goenv..." git init -q git remote add origin https://github.com/syndbg/goenv.git @@ -78,13 +77,13 @@ ynh_go_install () { local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") git checkout -q "$git_latest_tag" ynh_go_try_bash_extension - goenv=$goenv_install_dir/bin/goenv + goenv=$GOENV_INSTALL_DIR/bin/goenv popd # Install or update xxenv-latest - mkdir -p "$goenv_install_dir/plugins/xxenv-latest" - pushd "$goenv_install_dir/plugins/xxenv-latest" - if ! [ -x "$goenv_install_dir/plugins/xxenv-latest/bin/goenv-latest" ]; then + mkdir -p "$GOENV_INSTALL_DIR/plugins/xxenv-latest" + pushd "$GOENV_INSTALL_DIR/plugins/xxenv-latest" + if ! [ -x "$GOENV_INSTALL_DIR/plugins/xxenv-latest/bin/goenv-latest" ]; then ynh_print_info "Downloading xxenv-latest..." git init -q git remote add origin https://github.com/momo-lab/xxenv-latest.git @@ -97,10 +96,10 @@ ynh_go_install () { popd # Enable caching - mkdir -p "${goenv_install_dir}/cache" + mkdir -p "${GOENV_INSTALL_DIR}/cache" # Create shims directory if needed - mkdir -p "${goenv_install_dir}/shims" + mkdir -p "${GOENV_INSTALL_DIR}/shims" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH @@ -109,7 +108,7 @@ ynh_go_install () { test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go # Install the requested version of Go - local final_go_version=$(goenv latest --print "$go_version") + local final_go_version=$("$goenv_latest_dir/bin/goenv-latest" --print "$go_version") ynh_print_info "Installation of Go-$final_go_version" goenv install --quiet --skip-existing "$final_go_version" 2>&1 @@ -122,8 +121,8 @@ ynh_go_install () { # Set environment for Go users echo "#goenv -export GOENV_ROOT=$goenv_install_dir -export PATH=\"$goenv_install_dir/bin:$PATH\" +export GOENV_ROOT=$GOENV_INSTALL_DIR +export PATH=\"$GOENV_INSTALL_DIR/bin:$PATH\" eval \"\$(goenv init -)\" #goenv" > /etc/profile.d/goenv.sh @@ -142,7 +141,7 @@ ynh_go_remove () { local go_version=$(ynh_app_setting_get --key="go_version") # Load goenv path in PATH - local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Go prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -182,7 +181,7 @@ _ynh_go_cleanup () { if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1` then ynh_print_info "Removing of Go-$installed_go_version" - $goenv_install_dir/bin/goenv uninstall --force "$installed_go_version" + $GOENV_INSTALL_DIR/bin/goenv uninstall --force "$installed_go_version" fi done @@ -191,7 +190,7 @@ _ynh_go_cleanup () { then # Remove goenv environment configuration ynh_print_info "Removing of goenv" - ynh_safe_rm "$goenv_install_dir" + ynh_safe_rm "$GOENV_INSTALL_DIR" ynh_safe_rm "/etc/profile.d/goenv.sh" fi } diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index 5fce19d0b..cd10e29d6 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -14,7 +14,7 @@ FIRST_CALL_TO_LOGROTATE="true" # Requires YunoHost version 2.6.4 or higher. ynh_config_add_logrotate() { - logfile="$1" + local logfile="${1:-}" set -o noglob if [[ -z "$logfile" ]]; then diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index e8eecb829..5e5b13f4e 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -1,18 +1,17 @@ #!/bin/bash -n_install_dir="/opt/node_n" -node_version_path="$n_install_dir/n/versions/node" +readonly N_INSTALL_DIR="/opt/node_n" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. -export N_PREFIX="$n_install_dir" +export N_PREFIX="$N_INSTALL_DIR" _ynh_load_nodejs_in_path_and_other_tweaks() { # Get the absolute path of this version of node - local nodejs_path="$node_version_path/$nodejs_version/bin" + nodejs_dir="$N_INSTALL_DIR/n/versions/node/$nodejs_version/bin" # Load the path of this version of node in $PATH - if [[ :$PATH: != *":$nodejs_path"* ]]; then - PATH="$nodejs_path:$PATH" + if [[ :$PATH: != *":$nodejs_dir"* ]]; then + PATH="$nodejs_dir:$PATH" fi # Export PATH such that it's available through sudo -E / ynh_exec_as $app @@ -46,11 +45,11 @@ ynh_nodejs_install() { [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_install" - # Create $n_install_dir - mkdir --parents "$n_install_dir" + # Create $N_INSTALL_DIR + mkdir --parents "$N_INSTALL_DIR" # Load n path in PATH - CLEAR_PATH="$n_install_dir/bin:$PATH" + CLEAR_PATH="$N_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of node prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -59,10 +58,10 @@ ynh_nodejs_install() { test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n # Install (or update if YunoHost vendor/ folder updated since last install) n - mkdir -p $n_install_dir/bin/ - cp "$YNH_HELPERS_DIR/vendor/n/n" $n_install_dir/bin/n + mkdir -p $N_INSTALL_DIR/bin/ + cp "$YNH_HELPERS_DIR/vendor/n/n" $N_INSTALL_DIR/bin/n # Tweak for n to understand it's installed in $N_PREFIX - ynh_replace --match="^N_PREFIX=\${N_PREFIX-.*}$" --replace="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --file="$n_install_dir/bin/n" + ynh_replace --match="^N_PREFIX=\${N_PREFIX-.*}$" --replace="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --file="$N_INSTALL_DIR/bin/n" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH @@ -80,16 +79,18 @@ ynh_nodejs_install() { fi # Find the last "real" version for this major version of node. - real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1) + real_nodejs_version=$(find $N_INSTALL_DIR/n/versions/node/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1) real_nodejs_version=$(basename $real_nodejs_version) # Create a symbolic link for this major version if the file doesn't already exist - if [ ! -e "$node_version_path/$nodejs_version" ]; then - ln --symbolic --force --no-target-directory $node_version_path/$real_nodejs_version $node_version_path/$nodejs_version + if [ ! -e "$N_INSTALL_DIR/n/versions/node/$nodejs_version" ]; then + ln --symbolic --force --no-target-directory \ + $N_INSTALL_DIR/n/versions/node/$real_nodejs_version \ + $N_INSTALL_DIR/n/versions/node/$nodejs_version fi # Store the ID of this app and the version of node requested for it - echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version" + echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$N_INSTALL_DIR/ynh_app_version" # Store nodejs_version into the config of this app ynh_app_setting_set --key=nodejs_version --value=$nodejs_version @@ -111,17 +112,16 @@ ynh_nodejs_remove() { [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_remove" # Remove the line for this app - sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version" + sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$N_INSTALL_DIR/ynh_app_version" # If no other app uses this version of nodejs, remove it. - if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"; then - $n_install_dir/bin/n rm $nodejs_version + if ! grep --quiet "$nodejs_version" "$N_INSTALL_DIR/ynh_app_version"; then + $N_INSTALL_DIR/bin/n rm $nodejs_version fi # If no other app uses n, remove n - if [ ! -s "$n_install_dir/ynh_app_version" ]; then - ynh_safe_rm "$n_install_dir" - ynh_safe_rm "/usr/local/n" + if [ ! -s "$N_INSTALL_DIR/ynh_app_version" ]; then + ynh_safe_rm "$N_INSTALL_DIR" sed --in-place "/N_PREFIX/d" /root/.bashrc fi } diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index dd252cd45..a05768015 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -19,89 +19,68 @@ fi # # usage: ynh_config_add_phpfpm # -# This helper assumes the app has an conf/extra_php-fpm.conf snippet +# This will automatically generate an appropriate PHP-FPM configuration for this app. # -# The actual PHP configuration will be automatically generated, -# and your extra_php-fpm.conf will be appended (typically contains PHP upload limits) +# The resulting configuration will be deployed to the appropriate place: +# /etc/php/$php_version/fpm/pool.d/$app.conf # -# The resulting configuration will be deployed to the appropriate place, /etc/php/$php_version/fpm/pool.d/$app.conf +# If the app provides a conf/extra_php-fpm.conf template, it will be appended +# to the generated configuration. (In the vast majority of cases, this shouldnt +# be necessary) # -# Performance-related options in the PHP conf, such as : -# pm.max_children, pm.start_servers, pm.min_spare_servers pm.max_spare_servers -# are computed from two parameters called "usage" and "footprint" which can be set to low/medium/high. (cf details below) +# $php_version should be defined prior to calling this helper, but there should +# be no reason to manually set it, as it is automatically set by the apt +# helpers/resources when installing phpX.Y dependencies (PHP apps should at +# least install phpX.Y-fpm using the apt helper/resource) # -# If you wish to tweak those, please initialize the settings `fpm_usage` and `fpm_footprint` -# *prior* to calling this helper. Otherwise, "low" will be used as a default for both values. +# $php_group can be defined as a global (from _common.sh) if the worker +# processes should run with a different group than $app # -# Otherwise, if you want the user to have control over these, we encourage to create a config panel -# (which should ultimately be standardized by the core ...) +# Additional "pm" and "php_admin_value" settings which are meant to be possibly +# configurable by admins from a future standard config panel at some point, +# related to performance and availability of the app, for which tweaking may be +# required if the app is used by "plenty" of users and other memory/CPU load +# considerations.... # -# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM. -# So it will be used to defined 'pm.max_children' -# A lower value for the footprint will allow more children for 'pm.max_children'. And so for -# 'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the -# value of 'pm.max_children' -# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores. +# If you have good reasons to be willing to use different +# defaults than the one set by this helper (while still allowing admin to +# override it) you should use `ynh_app_setting_set_default` # -# The usage value will defined the way php will handle the children for the pool. -# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the -# service is used, otherwise no child will stay alive. This config gives the lower footprint when the -# service is idle. But will use more proc since it has to start a child as soon it's used. -# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children -# equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request. -# The number of children can grow if needed. The footprint can stay low if the service is idle, but -# not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few -# children already available. -# Set as 'high', the process manager will be set at 'static'. There will be always as many children as -# 'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum -# RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many -# children ready to answer. +# - $php_upload_max_filezise: corresponds upload_max_filesize and post_max_size. Defaults to 50M +# - $php_process_management: corresponds to "pm" (ondemand, dynamic, static). Defaults to ondemand +# - $php_max_children: by default, computed from "total RAM" divided by 40, cf _default_php_max_children +# - $php_memory_limit: by default, 128M (from global php.ini) +# +# Note that if $php_process_management is set to "dynamic", then these +# variables MUST be defined prior to calling the helper (no default value) ... +# Check PHP-FPM's manual for more info on what these are (: ... +# +# - $php_start_servers +# - $php_min_spare_servers +# - $php_max_spare_servers # -# Requires YunoHost version 4.1.0 or higher. ynh_config_add_phpfpm() { - # ============ Argument parsing ============= - local -A args_array=([g]=group=) - local group - ynh_handle_getopts_args "$@" - group=${group:-} - # =========================================== - # If the PHP version changed, remove the old fpm conf - # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper) - # FIXME: so is this still needed @_@ - local old_php_version=$(ynh_app_setting_get --key=php_version) - if [ -n "$old_php_version" ] && [ "$old_php_version" != "$php_version" ]; then - local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) - local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" + [[ -n "${php_version:-}" ]] || ynh_die "\$php_version should be defined prior to calling ynh_config_add_phpfpm. You should not need to define it manually, it is automatically set by the apt helper when installing the phpX.Y- depenencies" - if [[ -f "$old_php_finalphpconf" ]] - then - ynh_backup_if_checksum_is_different "$old_php_finalphpconf" - ynh_remove_fpm_config - fi - fi + # Apps may define $php_group as a global (e.g. from _common.sh) to change this + # (this is not meant to be overridable by users) + local php_group=${php_group:-$app} - local fpm_service="php${php_version}-fpm" - local fpm_config_dir="/etc/php/$php_version/fpm" + # Meant to be overridable by users from a standard config panel at some point ... + # Apps willing to tweak these should use ynh_setting_set_default_value (in install and upgrade?) + # + local php_upload_max_filesize=${php_upload_max_filesize:-50M} + local php_process_management=${php_process_management:-ondemand} # alternatively 'dynamic' or 'static' + local php_max_children=${php_max_children:-$(_default_php_max_children)} + local php_memory_limit=${php_memory_limit:-128M} # default value is from global php.ini - # Create the directory for FPM pools - mkdir --parents "$fpm_config_dir/pool.d" - - # FIXME: zzzz do we really need those ... - ynh_app_setting_set --key=fpm_config_dir --value="$fpm_config_dir" - ynh_app_setting_set --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --key=php_version --value=$php_version - - # Define the values to use for the configuration of PHP. - _ynh_get_scalable_phpfpm - - local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app") - local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" - echo " + local phpfpm_template=$(mktemp) + cat << EOF > $phpfpm_template [__APP__] user = __APP__ -group = __PHPFPM_GROUP__ +group = __PHP_GROUP__ chdir = __INSTALL_DIR__ @@ -109,39 +88,48 @@ listen = /var/run/php/php__PHP_VERSION__-fpm-__APP__.sock listen.owner = www-data listen.group = www-data -pm = __PHP_PM__ +pm = __PHP_PROCESS_MANAGEMENT__ pm.max_children = __PHP_MAX_CHILDREN__ pm.max_requests = 500 request_terminate_timeout = 1d -" >"$phpfpm_path" - if [ "$php_pm" = "dynamic" ]; then - echo " +EOF + if [ "$php_process_management" = "dynamic" ]; then + cat << EOF >> $phpfpm_template pm.start_servers = __PHP_START_SERVERS__ pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__ pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__ -" >>"$phpfpm_path" - - elif [ "$php_pm" = "ondemand" ]; then - echo " +EOF + elif [ "$php_process_management" = "ondemand" ]; then + cat << EOF >> $phpfpm_template pm.process_idle_timeout = 10s -" >>"$phpfpm_path" +EOF fi - # Concatene the extra config. + cat << EOF >> $phpfpm_template +php_admin_value[upload_max_filesize] = __PHP_UPLOAD_MAX_FILESIZE__ +php_admin_value[post_max_size] = __PHP_UPLOAD_MAX_FILESIZE__ +php_admin_value[memory_limit] = __PHP_MEMORY_LIMIT__ +EOF + + # Concatene the extra config if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then - cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path" + cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_template" fi - ynh_config_add --template="$phpfpm_path" --destination="$fpm_config_dir/pool.d/$app.conf" + # Make sure the fpm pool dir exists + mkdir --parents "/etc/php/$php_version/fpm/pool.d" + # And hydrate configuration + ynh_config_add --template="$phpfpm_template" --destination="/etc/php/$php_version/fpm/pool.d/$app.conf" # Validate that the new php conf doesn't break php-fpm entirely if ! php-fpm${php_version} --test 2>/dev/null; then php-fpm${php_version} --test || true - ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" + ynh_safe_rm "/etc/php/$php_version/fpm/pool.d/$app.conf" ynh_die "The new configuration broke php-fpm?" fi - ynh_systemctl --service=$fpm_service --action=reload + + ynh_systemctl --service=php${php_version}-fpm --action=reload } # Remove the dedicated PHP-FPM config @@ -150,125 +138,31 @@ pm.process_idle_timeout = 10s # # Requires YunoHost version 2.7.2 or higher. ynh_config_remove_phpfpm() { - local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) - - ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" + ynh_safe_rm "/etc/php/$php_version/fpm/pool.d/$app.conf" ynh_systemctl --service="php${php_version}-fpm" --action=reload } -# Define the values to configure PHP-FPM -# -# [internal] -# -# usage: _ynh_get_scalable_phpfpm -# Footprint can be defined via the "fpm_footprint", to be set prior to calling this helper -# low - Less than 20 MB of RAM by pool. -# medium - Between 20 MB and 40 MB of RAM by pool. -# high - More than 40 MB of RAM by pool. -# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value. -# To have this value, use the following command and stress the service. -# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP -# -# Usage can be defined via the "fpm_usage", to be set prior to calling this helper -# low - Personal usage, behind the SSO. -# medium - Low usage, few people or/and publicly accessible. -# high - High usage, frequently visited website. -# -_ynh_get_scalable_phpfpm() { - - # If no usage provided, default to the value existing in setting ... or to low - local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) - local usage=${fpm_usage_in_setting:-low} - ynh_app_setting_set --key=fpm_usage --value=$usage - - # If no footprint provided, default to the value existing in setting ... or to low - local fpm_footprint_in_setting=$(ynh_app_setting_get --key=fpm_footprint) - local footprint=${fpm_footprint_in_setting:-low} - ynh_app_setting_set --key=fpm_footprint --value=$footprint - - # Set all characters as lowercase - if [ "$footprint" = "low" ]; then - footprint=20 - elif [ "$footprint" = "medium" ]; then - footprint=35 - elif [ "$footprint" = "high" ]; then - footprint=50 - fi - - # Define the factor to determine min_spare_servers - # to avoid having too few children ready to start for heavy apps - if [ $footprint -le 20 ]; then - min_spare_servers_factor=8 - elif [ $footprint -le 35 ]; then - min_spare_servers_factor=5 - else - min_spare_servers_factor=3 - fi - - # Define the way the process manager handle child processes. - if [ "$usage" = "low" ]; then - php_pm=ondemand - elif [ "$usage" = "medium" ]; then - php_pm=dynamic - elif [ "$usage" = "high" ]; then - php_pm=static - else - ynh_die "Does not recognize '$usage' as an usage value." - fi - - # Get the total of RAM available, except swap. - local max_ram=$(ynh_get_ram --total) - - at_least_one() { - # Do not allow value below 1 - if [ $1 -le 0 ]; then - echo 1 - else - echo $1 - fi - } - - # Define pm.max_children - # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. - # So if PHP-FPM start the maximum of children, it won't exceed half of the ram. - php_max_children=$(($max_ram / 2 / $footprint)) - # If process manager is set as static, use half less children. - # Used as static, there's always as many children as the value of pm.max_children - if [ "$php_pm" = "static" ]; then - php_max_children=$(($php_max_children / 2)) - fi - php_max_children=$(at_least_one $php_max_children) +_default_php_max_children() { + # Get the total of RAM available + local total_ram=$(ynh_get_ram --total) + # The value of pm.max_children is the total amount of ram divide by 2, + # divide again by 20MB (= a default, classic worker footprint) This is + # designed such that if PHP-FPM start the maximum of children, it won't + # exceed half of the ram. + local php_max_children="$(($total_ram / 40))" + # Make sure we get at least max_children = 1 + if [ $php_max_children -le 0 ]; then + php_max_children=1 # To not overload the proc, limit the number of children to 4 times the number of cores. - local core_number=$(nproc) - local max_proc=$(($core_number * 4)) - if [ $php_max_children -gt $max_proc ]; then - php_max_children=$max_proc + elif [ $php_max_children -gt "$(($(nproc) * 4))" ]; then + php_max_children="$(($(nproc) * 4))" fi - # Get a potential forced value for php_max_children - local php_forced_max_children=$(ynh_app_setting_get --key=php_forced_max_children) - if [ -n "$php_forced_max_children" ]; then - php_max_children=$php_forced_max_children - fi - - if [ "$php_pm" = "dynamic" ]; then - # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager - php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor)) - php_min_spare_servers=$(at_least_one $php_min_spare_servers) - - php_max_spare_servers=$(($php_max_children / 2)) - php_max_spare_servers=$(at_least_one $php_max_spare_servers) - - php_start_servers=$(($php_min_spare_servers + ($php_max_spare_servers - $php_min_spare_servers) / 2)) - php_start_servers=$(at_least_one $php_start_servers) - else - php_min_spare_servers=0 - php_max_spare_servers=0 - php_start_servers=0 - fi + echo "$php_max_children" } + # Execute a command with Composer # # Will use $install_dir as workdir unless $composer_workdir exists (but that shouldnt be necessary) diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 713743ec0..ffaeafbf3 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -81,6 +81,8 @@ ynh_psql_dump_db() { # | arg: pwd - the password to identify user by # ynh_psql_create_user() { + local user=$1 + local pwd=$2 sudo --login --user=postgres psql <<< "CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" } diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index dae25c6fd..c33967a9f 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -1,20 +1,19 @@ #!/bin/bash -rbenv_install_dir="/opt/rbenv" -ruby_version_path="$rbenv_install_dir/versions" +readonly RBENV_INSTALL_DIR="/opt/rbenv" # RBENV_ROOT is the directory of rbenv, it needs to be loaded as a environment variable. -export RBENV_ROOT="$rbenv_install_dir" -export rbenv_root="$rbenv_install_dir" +export RBENV_ROOT="$RBENV_INSTALL_DIR" +export rbenv_root="$RBENV_INSTALL_DIR" _ynh_load_ruby_in_path_and_other_tweaks() { # Get the absolute path of this version of Ruby - local ruby_path="$ruby_version_path/$app/bin" + ruby_dir="$RBENV_INSTALL_DIR/versions/$app/bin" # Load the path of this version of ruby in $PATH - if [[ :$PATH: != *":$ruby_path"* ]]; then - PATH="$ruby_path:$PATH" + if [[ :$PATH: != *":$ruby_dir"* ]]; then + PATH="$ruby_dir:$PATH" fi # Export PATH such that it's available through sudo -E / ynh_exec_as $app @@ -26,7 +25,7 @@ _ynh_load_ruby_in_path_and_other_tweaks() { # Sets the local application-specific Ruby version pushd ${install_dir} - $rbenv_install_dir/bin/rbenv local $ruby_version + $RBENV_INSTALL_DIR/bin/rbenv local $ruby_version popd } @@ -55,7 +54,7 @@ ynh_ruby_install () { [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_install" # Load rbenv path in PATH - local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + local CLEAR_PATH="$RBENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Ruby prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -64,8 +63,8 @@ ynh_ruby_install () { test -x /usr/bin/ruby && mv /usr/bin/ruby /usr/bin/ruby_rbenv # Install or update rbenv - mkdir -p $rbenv_install_dir - rbenv="$(command -v rbenv $rbenv_install_dir/bin/rbenv | grep "$rbenv_install_dir/bin/rbenv" | head -1)" + mkdir -p $RBENV_INSTALL_DIR + rbenv="$(command -v rbenv $RBENV_INSTALL_DIR/bin/rbenv | grep "$RBENV_INSTALL_DIR/bin/rbenv" | head -1)" if [ -n "$rbenv" ]; then pushd "${rbenv%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/rbenv/rbenv.git"; then @@ -75,30 +74,30 @@ ynh_ruby_install () { else echo "Reinstalling rbenv..." cd .. - ynh_safe_rm $rbenv_install_dir - mkdir -p $rbenv_install_dir - cd $rbenv_install_dir + ynh_safe_rm $RBENV_INSTALL_DIR + mkdir -p $RBENV_INSTALL_DIR + cd $RBENV_INSTALL_DIR git init -q git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 git checkout -q -b master origin/master ynh_ruby_try_bash_extension - rbenv=$rbenv_install_dir/bin/rbenv + rbenv=$RBENV_INSTALL_DIR/bin/rbenv fi popd else echo "Installing rbenv..." - pushd $rbenv_install_dir + pushd $RBENV_INSTALL_DIR git init -q git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 git checkout -q -b master origin/master ynh_ruby_try_bash_extension - rbenv=$rbenv_install_dir/bin/rbenv + rbenv=$RBENV_INSTALL_DIR/bin/rbenv popd fi - mkdir -p "${rbenv_install_dir}/plugins" + mkdir -p "${RBENV_INSTALL_DIR}/plugins" - ruby_build="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-install rbenv-install | head -1)" + ruby_build="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-install rbenv-install | head -1)" if [ -n "$ruby_build" ]; then pushd "${ruby_build%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/rbenv/ruby-build.git"; then @@ -108,10 +107,10 @@ ynh_ruby_install () { popd else echo "Installing ruby-build..." - git clone -q https://github.com/rbenv/ruby-build.git "${rbenv_install_dir}/plugins/ruby-build" + git clone -q https://github.com/rbenv/ruby-build.git "${RBENV_INSTALL_DIR}/plugins/ruby-build" fi - rbenv_alias="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)" + rbenv_alias="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)" if [ -n "$rbenv_alias" ]; then pushd "${rbenv_alias%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/tpope/rbenv-aliases.git"; then @@ -121,10 +120,10 @@ ynh_ruby_install () { popd else echo "Installing rbenv-aliases..." - git clone -q https://github.com/tpope/rbenv-aliases.git "${rbenv_install_dir}/plugins/rbenv-aliase" + git clone -q https://github.com/tpope/rbenv-aliases.git "${RBENV_INSTALL_DIR}/plugins/rbenv-aliase" fi - rbenv_latest="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)" + rbenv_latest="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)" if [ -n "$rbenv_latest" ]; then pushd "${rbenv_latest%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then @@ -134,14 +133,14 @@ ynh_ruby_install () { popd else echo "Installing xxenv-latest..." - git clone -q https://github.com/momo-lab/xxenv-latest.git "${rbenv_install_dir}/plugins/xxenv-latest" + git clone -q https://github.com/momo-lab/xxenv-latest.git "${RBENV_INSTALL_DIR}/plugins/xxenv-latest" fi # Enable caching - mkdir -p "${rbenv_install_dir}/cache" + mkdir -p "${RBENV_INSTALL_DIR}/cache" # Create shims directory if needed - mkdir -p "${rbenv_install_dir}/shims" + mkdir -p "${RBENV_INSTALL_DIR}/shims" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH @@ -175,8 +174,8 @@ ynh_ruby_install () { # Set environment for Ruby users echo "#rbenv -export RBENV_ROOT=$rbenv_install_dir -export PATH=\"$rbenv_install_dir/bin:$PATH\" +export RBENV_ROOT=$RBENV_INSTALL_DIR +export PATH=\"$RBENV_INSTALL_DIR/bin:$PATH\" eval \"\$(rbenv init -)\" #rbenv" > /etc/profile.d/rbenv.sh @@ -192,10 +191,11 @@ eval \"\$(rbenv init -)\" # # usage: ynh_ruby_remove ynh_ruby_remove () { - local ruby_version=$(ynh_app_setting_get --key=ruby_version) + + [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_remove" # Load rbenv path in PATH - local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + local CLEAR_PATH="$RBENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Ruby prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -235,7 +235,7 @@ _ynh_ruby_cleanup () { if ! echo ${required_ruby_versions} | grep -q "${installed_ruby_version}" then echo "Removing Ruby-$installed_ruby_version" - $rbenv_install_dir/bin/rbenv uninstall --force $installed_ruby_version + $RBENV_INSTALL_DIR/bin/rbenv uninstall --force $installed_ruby_version fi done @@ -244,7 +244,7 @@ _ynh_ruby_cleanup () { then # Remove rbenv environment configuration echo "Removing rbenv" - ynh_safe_rm "$rbenv_install_dir" + ynh_safe_rm "$RBENV_INSTALL_DIR" ynh_safe_rm "/etc/profile.d/rbenv.sh" fi } diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 0d737491e..d8053066d 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -42,6 +42,41 @@ ynh_app_setting_set() { ynh_app_setting "set" "$app" "$key" "$value" } +# Set an application setting but only if the "$key" variable ain't set yet +# +# Note that it doesn't just define the setting but ALSO define the $foobar variable +# +# Hence it's meant as a replacement for this legacy overly complex syntax: +# +# if [ -z "${foo:-}" ] +# then +# foo="bar" +# ynh_app_setting_set --key="foo" --value="$foo" +# fi +# +# usage: ynh_app_setting_set_default --app=app --key=key --value=value +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting name to set +# | arg: -v, --value= - the default setting value to set +# +# Requires YunoHost version 11.1.16 or higher. +ynh_app_setting_set_default() { + # ============ Argument parsing ============= + local _globalapp=${app-:} + local -A args_array=([a]=app= [k]=key= [v]=value=) + local app + local key + local value + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + # =========================================== + + if [ -z "${!key:-}" ]; then + eval $key=\$value + ynh_app_setting "set" "$app" "$key" "$value" + fi +} + # Delete an application setting # # usage: ynh_app_setting_delete --app=app --key=key @@ -68,6 +103,8 @@ ynh_app_setting_delete() { # [internal] # ynh_app_setting() { + # Trick to only re-enable debugging if it was set before + local xtrace_enable=$(set +o | grep xtrace) set +o xtrace # set +x ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <&2 fi @@ -156,6 +162,13 @@ ynh_systemctl() { ynh_print_warn "===" tail --lines=$length "$log_path" >&2 fi + + # If we tried to reload/start/restart the service but systemctl consider it to be still inactive/broken, then handle it as a failure + if ([ "$action" == "reload" ] || [ "$action" == "start" ] || [ "$action" == "restart" ]) && ! systemctl --quiet is-active $service + then + _ynh_clean_check_starting + return 1 + fi fi _ynh_clean_check_starting fi diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index 227653893..1ce0e78de 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -174,9 +174,9 @@ ynh_system_user_delete() { # Execute a command after sudoing as $app # -# Note that exported bash env variables are kept (using -E option of sudo) +# Note that the $PATH variable is preserved (using --preserve-env=PATH) # # usage: ynh_exec_as_app COMMAND [ARG ...] ynh_exec_as_app() { - sudo -E -u"$app" "$@" + sudo --preserve-env=PATH -u "$app" "$@" } diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index e6eb2e88b..c46c41d7c 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -158,7 +158,6 @@ ynh_setup_source() { local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" - local src_sumprg="sha256sum" local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" src_in_subdir=${src_in_subdir:-true} @@ -198,9 +197,6 @@ ynh_setup_source() { ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" fi - # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... - local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" - # Gotta use this trick with 'dirname' because source_id may contain slashes x_x mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" @@ -208,14 +204,10 @@ ynh_setup_source() { if [ "$src_format" = "docker" ]; then src_platform="${src_platform:-"linux/$YNH_ARCH"}" else - if test -e "$local_src"; then - cp $local_src $src_filename - fi - [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" # If the file was prefetched but somehow doesn't match the sum, rm and redownload it - if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status then rm -f "$src_filename" fi @@ -233,9 +225,9 @@ ynh_setup_source() { fi # Check the control sum - if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status then - local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)" local actual_size="$(du -hs ${src_filename} | cut --fields=1)" rm -f ${src_filename} ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." @@ -468,7 +460,7 @@ ynh_read_manifest() { # # Requires YunoHost version 3.5.0 or higher. ynh_app_upstream_version() { - echo "${$YNH_APP_MANIFEST_VERSION/~ynh*/}" + echo "${YNH_APP_MANIFEST_VERSION/~ynh*/}" } # Return 0 if the "upstream" part of the version changed, or 1 otherwise (ie only the ~ynh suffix changed) diff --git a/locales/eu.json b/locales/eu.json index c21a18795..2b84b0763 100644 --- a/locales/eu.json +++ b/locales/eu.json @@ -601,8 +601,8 @@ "restore_hook_unavailable": "'{part}'-(e)rako lehengoratze agindua ez dago erabilgarri ez sisteman ezta fitxategian ere", "restore_cleaning_failed": "Ezin izan dira lehengoratzeko behin-behineko fitxategiak ezabatu", "restore_confirm_yunohost_installed": "Ziur al zaude dagoeneko instalatuta dagoen sistema lehengoratu nahi duzula? [{answers}]", - "restore_may_be_not_enough_disk_space": "Badirudi zure sistemak ez duela nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasuneko tartea: {margin} B)", - "restore_not_enough_disk_space": "Ez dago nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasuneko tartea: {margin} B)", + "restore_may_be_not_enough_disk_space": "Badirudi zure sistemak ez duela nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasun-tartea: {margin} B)", + "restore_not_enough_disk_space": "Ez dago nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasun-tartea: {margin} B)", "restore_running_hooks": "Lehengoratzeko 'hook'ak exekutatzen…", "restore_system_part_failed": "Ezinezkoa izan da sistemaren '{part}' atala lehengoratzea", "server_reboot": "Zerbitzaria berrabiaraziko da", diff --git a/locales/sk.json b/locales/sk.json index a8af9df99..961555f8a 100644 --- a/locales/sk.json +++ b/locales/sk.json @@ -265,5 +265,18 @@ "service_description_rspamd": "Filtruje spam a iné funkcie týkajúce sa e-mailu", "log_letsencrypt_cert_renew": "Obnoviť '{}' certifikát Let's Encrypt", "domain_config_cert_summary_selfsigned": "UPOZORNENIE: Aktuálny certifikát je vlastnoručne podpísaný. Prehliadače budú návštevníkom zobrazovať strašidelné varovanie!", - "global_settings_setting_ssowat_panel_overlay_enabled": "Povoliť malú štvorcovú ikonu portálu „YunoHost“ na aplikáciach" + "global_settings_setting_ssowat_panel_overlay_enabled": "Povoliť malú štvorcovú ikonu portálu „YunoHost“ na aplikáciach", + "domain_config_mail_out": "Odchádzajúce e-maily", + "domain_config_default_app": "Predvolená aplikácia", + "domain_config_xmpp_help": "Pozor: niektoré funkcie XMPP vyžadujú aktualizáciu vašich DNS záznamov a obnovenie Lets Encrypt certifikátu pred tým, ako je ich možné zapnúť", + "domain_config_default_app_help": "Návštevníci budú pri návšteve tejto domény automaticky presmerovaní na túto doménu. Ak nenastavíte žiadnu aplikáciu, zobrazí sa stránka s prihlasovacím formulárom na portál.", + "registrar_infos": "Informácie o registrátorovi", + "domain_dns_registrar_managed_in_parent_domain": "Táto doména je subdoména {parent_domain_link}. Nastavenie DNS registrátora je spravovaná v konfiguračnom paneli {parent_domain}.", + "log_letsencrypt_cert_install": "Inštalovať certifikát Let's Encrypt na doménu '{}'", + "domain_config_cert_no_checks": "Ignorovať kontroly diagnostiky", + "domain_config_cert_install": "Nainštalovať certifikát Let's Encrypt", + "domain_config_mail_in": "Prichádzajúce e-maily", + "domain_config_cert_summary": "Stav certifikátu", + "domain_config_xmpp": "Krátke správy (XMPP)", + "log_app_makedefault": "Nastaviť '{}' ako predvolenú aplikáciu" } diff --git a/src/app.py b/src/app.py index 3051fbbf2..b729eab19 100644 --- a/src/app.py +++ b/src/app.py @@ -2992,19 +2992,43 @@ def _make_environment_for_app_script( # If packaging format v2, load all settings if manifest["packaging_format"] >= 2 or force_include_app_settings: env_dict["app"] = app + data_to_redact = [] + prefixes_or_suffixes_to_redact = [ + "pwd", + "pass", + "passwd", + "password", + "passphrase", + "secret", + "key", + "token", + ] + for setting_name, setting_value in _get_app_settings(app).items(): # Ignore special internal settings like checksum__ # (not a huge deal to load them but idk...) if setting_name.startswith("checksum__"): continue - env_dict[setting_name] = str(setting_value) + setting_value = str(setting_value) + env_dict[setting_name] = setting_value + + # Check if we should redact this setting value + # (the check on the setting length exists to prevent stupid stuff like redacting empty string or something which is actually just 0/1, true/false, ... + if len(setting_value) > 6 and any( + setting_name.startswith(p) or setting_name.endswith(p) + for p in prefixes_or_suffixes_to_redact + ): + data_to_redact.append(setting_value) # Special weird case for backward compatibility... # 'path' was loaded into 'path_url' ..... if "path" in env_dict: env_dict["path_url"] = env_dict["path"] + for operation_logger in OperationLogger._instances: + operation_logger.data_to_redact.extend(data_to_redact) + return env_dict diff --git a/src/log.py b/src/log.py index 818d0c1a9..7950fca78 100755 --- a/src/log.py +++ b/src/log.py @@ -45,12 +45,18 @@ LOG_FILE_EXT = ".log" BORING_LOG_LINES = [ r"set [+-]x$", r"set [+-]o xtrace$", + r"\+ set \+o$", + r"\+ grep xtrace$", + r"local 'xtrace_enable=", r"set [+-]o errexit$", r"set [+-]o nounset$", r"trap '' EXIT", r"local \w+$", r"local exit_code=(1|0)$", r"local legacy_args=.*$", + r"local _globalapp=.*$", + r"local checksum_setting_name=.*$", + r"ynh_app_setting ", # (note the trailing space to match the "low level" one called by other setting helpers) r"local -A args_array$", r"args_array=.*$", r"ret_code=1", @@ -62,8 +68,17 @@ BORING_LOG_LINES = [ r"\[?\['? -n '' '?\]\]?$", r"rm -rf /var/cache/yunohost/download/$", r"type -t ynh_clean_setup$", + r"DEBUG - \+ unset \S+$", r"DEBUG - \+ echo '", + r"DEBUG - \+ LC_ALL=C$", + r"DEBUG - \+ DEBIAN_FRONTEND=noninteractive$", r"DEBUG - \+ exit (1|0)$", + r"DEBUG - \+ app=\S+$", + r"DEBUG - \+\+ app=\S+$", + r"DEBUG - \+\+ jq -r .\S+$", + r"DEBUG - \+\+ sed 's/\^null\$//'$", + "DEBUG - \\+ sed --in-place \\$'s\\\\001", + "DEBUG - \\+ sed --in-place 's\u0001.*$", ] diff --git a/src/utils/resources.py b/src/utils/resources.py index b408fde47..3e41044ea 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -156,29 +156,29 @@ class AppResource: app_upstream_version = manager.current["version"].split("~")[0] # FIXME : should use packaging.version to properly parse / compare versions >_> - self.helpers_version = None + self.helpers_version: float = 0 if ( manager and manager.wanted and manager.wanted.get("integration", {}).get("helpers_version") ): - self.helpers_version = manager.wanted.get("integration", {}).get( - "helpers_version" + self.helpers_version = float( + manager.wanted.get("integration", {}).get("helpers_version") ) elif ( manager and manager.current and manager.current.get("integration", {}).get("helpers_version") ): - self.helpers_version = manager.current.get("integration", {}).get( - "helpers_version" + self.helpers_version = float( + manager.current.get("integration", {}).get("helpers_version") ) elif manager and manager.wanted and manager.wanted.get("packaging_format"): - self.helpers_version = str(manager.wanted.get("packaging_format")) + self.helpers_version = float(manager.wanted.get("packaging_format")) elif manager and manager.current and manager.current.get("packaging_format"): - self.helpers_version = str(manager.current.get("packaging_format")) + self.helpers_version = float(manager.current.get("packaging_format")) if not self.helpers_version: - self.helpers_version = "1" + self.helpers_version = 1.0 replacements: dict[str, str] = { "__APP__": self.app, @@ -1208,7 +1208,7 @@ class AptDependenciesAppResource(AppResource): def provision_or_update(self, context: Dict = {}): - if float(self.helpers_version) >= 2.1: + if self.helpers_version >= 2.1: ynh_apt_install_dependencies = "ynh_apt_install_dependencies" ynh_apt_install_dependencies_from_extra_repository = ( "ynh_apt_install_dependencies_from_extra_repository" @@ -1234,7 +1234,7 @@ class AptDependenciesAppResource(AppResource): self._run_script("provision_or_update", script) def deprovision(self, context: Dict = {}): - if float(self.helpers_version) >= 2.1: + if self.helpers_version >= 2.1: ynh_apt_remove_dependencies = "ynh_apt_remove_dependencies" else: ynh_apt_remove_dependencies = "ynh_remove_app_dependencies"