From 1c7e139c74b82867f894286eb298096d375cb575 Mon Sep 17 00:00:00 2001 From: Chris Vogel Date: Sat, 18 May 2024 14:03:32 +0200 Subject: [PATCH 001/146] Update main.cf to allow aliases for sender addresses of apps If an app registers the sender address _app@doma.in_ **and** a yunohost account configured an alias for the same address the app will not be able to send emails anymore. postfix asks the first map defined in smtpd_sender_login_maps and finds that the address _app@doma.in_ can be used as a sender address by the yunohost account having configured the alias and **then stops and doesn't look up the second list** for registered apps. The unionmap instructs postfix to join a list from both sources and then return the match from that joined list which would then contain _yunohost_account_having_registered_alias, appname_ for the lookup of _app@doma.in_. This allows the yunohost account having registered the alias and the app being registered to use the sender address to send email using that sender **and makes it possible to receive replies to the emails going out from the app**. Reference: https://serverfault.com/questions/948362/postfix-multiple-smtpd-sender-login-maps --- conf/postfix/main.cf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/postfix/main.cf b/conf/postfix/main.cf index 2867a04c2..957907dd3 100644 --- a/conf/postfix/main.cf +++ b/conf/postfix/main.cf @@ -107,12 +107,12 @@ virtual_alias_domains = virtual_minimum_uid = 100 virtual_uid_maps = static:vmail virtual_gid_maps = static:mail -smtpd_sender_login_maps= +smtpd_sender_login_maps = unionmap:{ # Regular Yunohost accounts ldap:/etc/postfix/ldap-accounts.cf, # Extra maps for app system users who need to send emails hash:/etc/postfix/app_senders_login_maps - +} # Dovecot LDA virtual_transport = dovecot From 204800e87817077af07279f5fa158122826ed104 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 17:49:30 +0200 Subject: [PATCH 002/146] helpers: copy v1 helpers to new v2.1 --- helpers/helpers.v2.1.d/apps | 215 +++ helpers/helpers.v2.1.d/apt | 636 ++++++ helpers/helpers.v2.1.d/backup | 507 +++++ helpers/helpers.v2.1.d/config | 354 ++++ helpers/helpers.v2.1.d/fail2ban | 136 ++ helpers/helpers.v2.1.d/getopts | 215 +++ helpers/helpers.v2.1.d/go | 247 +++ helpers/helpers.v2.1.d/hardware | 105 + helpers/helpers.v2.1.d/logging | 347 ++++ helpers/helpers.v2.1.d/logrotate | 103 + helpers/helpers.v2.1.d/mongodb | 355 ++++ helpers/helpers.v2.1.d/multimedia | 103 + helpers/helpers.v2.1.d/mysql | 249 +++ helpers/helpers.v2.1.d/network | 132 ++ helpers/helpers.v2.1.d/nginx | 65 + helpers/helpers.v2.1.d/nodejs | 241 +++ helpers/helpers.v2.1.d/permission | 399 ++++ helpers/helpers.v2.1.d/php | 583 ++++++ helpers/helpers.v2.1.d/postgresql | 310 +++ helpers/helpers.v2.1.d/redis | 39 + helpers/helpers.v2.1.d/ruby | 306 +++ helpers/helpers.v2.1.d/setting | 160 ++ helpers/helpers.v2.1.d/string | 151 ++ helpers/helpers.v2.1.d/systemd | 189 ++ helpers/helpers.v2.1.d/user | 196 ++ helpers/helpers.v2.1.d/utils | 1104 +++++++++++ .../vendor/docker-image-extract/LICENSE | 19 + .../vendor/docker-image-extract/README.md | 1 + .../docker-image-extract/docker-image-extract | 288 +++ helpers/helpers.v2.1.d/vendor/n/LICENSE | 21 + helpers/helpers.v2.1.d/vendor/n/README.md | 1 + helpers/helpers.v2.1.d/vendor/n/n | 1713 +++++++++++++++++ 32 files changed, 9490 insertions(+) create mode 100644 helpers/helpers.v2.1.d/apps create mode 100644 helpers/helpers.v2.1.d/apt create mode 100644 helpers/helpers.v2.1.d/backup create mode 100644 helpers/helpers.v2.1.d/config create mode 100644 helpers/helpers.v2.1.d/fail2ban create mode 100644 helpers/helpers.v2.1.d/getopts create mode 100644 helpers/helpers.v2.1.d/go create mode 100644 helpers/helpers.v2.1.d/hardware create mode 100644 helpers/helpers.v2.1.d/logging create mode 100644 helpers/helpers.v2.1.d/logrotate create mode 100644 helpers/helpers.v2.1.d/mongodb create mode 100644 helpers/helpers.v2.1.d/multimedia create mode 100644 helpers/helpers.v2.1.d/mysql create mode 100644 helpers/helpers.v2.1.d/network create mode 100644 helpers/helpers.v2.1.d/nginx create mode 100644 helpers/helpers.v2.1.d/nodejs create mode 100644 helpers/helpers.v2.1.d/permission create mode 100644 helpers/helpers.v2.1.d/php create mode 100644 helpers/helpers.v2.1.d/postgresql create mode 100644 helpers/helpers.v2.1.d/redis create mode 100644 helpers/helpers.v2.1.d/ruby create mode 100644 helpers/helpers.v2.1.d/setting create mode 100644 helpers/helpers.v2.1.d/string create mode 100644 helpers/helpers.v2.1.d/systemd create mode 100644 helpers/helpers.v2.1.d/user create mode 100644 helpers/helpers.v2.1.d/utils create mode 100644 helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE create mode 100644 helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md create mode 100755 helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract create mode 100644 helpers/helpers.v2.1.d/vendor/n/LICENSE create mode 100644 helpers/helpers.v2.1.d/vendor/n/README.md create mode 100755 helpers/helpers.v2.1.d/vendor/n/n diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps new file mode 100644 index 000000000..81a5717eb --- /dev/null +++ b/helpers/helpers.v2.1.d/apps @@ -0,0 +1,215 @@ +#!/bin/bash + +# Install others YunoHost apps +# +# usage: ynh_install_apps --apps="appfoo?domain=domain.foo&path=/foo appbar?domain=domain.bar&path=/bar&admin=USER&language=fr&is_public=1&pass?word=pass&port=666" +# | arg: -a, --apps= - apps to install +# +# Requires YunoHost version *.*.* or higher. +ynh_install_apps() { + # Declare an array to define the options of this helper. + local legacy_args=a + local -A args_array=([a]=apps=) + local apps + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Split the list of apps in an array + local apps_list=($(echo $apps | tr " " "\n")) + local apps_dependencies="" + + # For each app + for one_app_and_its_args in "${apps_list[@]}" + do + # Retrieve the name of the app (part before ?) + local one_app=$(cut -d "?" -f1 <<< "$one_app_and_its_args") + [ -z "$one_app" ] && ynh_die --message="You didn't provided a YunoHost app to install" + + yunohost tools update apps + + # Installing or upgrading the app depending if it's installed or not + if ! yunohost app list --output-as json --quiet | jq -e --arg id $one_app '.apps[] | select(.id == $id)' >/dev/null + then + # Retrieve the arguments of the app (part after ?) + local one_argument="" + if [[ "$one_app_and_its_args" == *"?"* ]]; then + one_argument=$(cut -d "?" -f2- <<< "$one_app_and_its_args") + one_argument="--args $one_argument" + fi + + # Install the app with its arguments + yunohost app install $one_app $one_argument + else + # Upgrade the app + yunohost app upgrade $one_app + fi + + if [ ! -z "$apps_dependencies" ] + then + apps_dependencies="$apps_dependencies, $one_app" + else + apps_dependencies="$one_app" + fi + done + + ynh_app_setting_set --app=$app --key=apps_dependencies --value="$apps_dependencies" +} + +# Remove other YunoHost apps +# +# Other YunoHost apps will be removed only if no other apps need them. +# +# usage: ynh_remove_apps +# +# Requires YunoHost version *.*.* or higher. +ynh_remove_apps() { + # Retrieve the apps dependencies of the app + local apps_dependencies=$(ynh_app_setting_get --app=$app --key=apps_dependencies) + ynh_app_setting_delete --app=$app --key=apps_dependencies + + if [ ! -z "$apps_dependencies" ] + then + # Split the list of apps dependencies in an array + local apps_dependencies_list=($(echo $apps_dependencies | tr ", " "\n")) + + # For each apps dependencies + for one_app in "${apps_dependencies_list[@]}" + do + # Retrieve the list of installed apps + local installed_apps_list=$(yunohost app list --output-as json --quiet | jq -r .apps[].id) + local required_by="" + local installed_app_required_by="" + + # For each other installed app + for one_installed_app in $installed_apps_list + do + # Retrieve the other apps dependencies + one_installed_apps_dependencies=$(ynh_app_setting_get --app=$one_installed_app --key=apps_dependencies) + if [ ! -z "$one_installed_apps_dependencies" ] + then + one_installed_apps_dependencies_list=($(echo $one_installed_apps_dependencies | tr ", " "\n")) + + # For each dependency of the other apps + for one_installed_app_dependency in "${one_installed_apps_dependencies_list[@]}" + do + if [[ $one_installed_app_dependency == $one_app ]]; then + required_by="$required_by $one_installed_app" + fi + done + fi + done + + # If $one_app is no more required + if [[ -z "$required_by" ]] + then + # Remove $one_app + ynh_print_info --message="Removing of $one_app" + yunohost app remove $one_app --purge + else + ynh_print_info --message="$one_app was not removed because it's still required by${required_by}" + fi + done + fi +} + +# Spawn a Bash shell with the app environment loaded +# +# usage: ynh_spawn_app_shell --app="app" +# | arg: -a, --app= - the app ID +# +# examples: +# ynh_spawn_app_shell --app="APP" <<< 'echo "$USER"' +# ynh_spawn_app_shell --app="APP" < /tmp/some_script.bash +# +# Requires YunoHost version 11.0.* or higher, and that the app relies on packaging v2 or higher. +# The spawned shell will have environment variables loaded and environment files sourced +# from the app's service configuration file (defaults to $app.service, overridable by the packager with `service` setting). +# If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings. +ynh_spawn_app_shell() { + # Declare an array to define the options of this helper. + local legacy_args=a + local -A args_array=([a]=app=) + local app + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Force Bash to be used to run this helper + if [[ ! $0 =~ \/?bash$ ]] + then + ynh_print_err --message="Please use Bash as shell" + exit 1 + fi + + # Make sure the app is installed + local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id)) + if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]] + then + ynh_print_err --message="$app is not in the apps list" + exit 1 + fi + + # Make sure the app has its own user + if ! id -u "$app" &>/dev/null; then + ynh_print_err --message="There is no \"$app\" system user" + exit 1 + fi + + # Make sure the app has an install_dir setting + local install_dir=$(ynh_app_setting_get --app=$app --key=install_dir) + if [ -z "$install_dir" ] + then + ynh_print_err --message="$app has no install_dir setting (does it use packaging format >=2?)" + exit 1 + fi + + # Load the app's service name, or default to $app + local service=$(ynh_app_setting_get --app=$app --key=service) + [ -z "$service" ] && service=$app; + + # Export HOME variable + export HOME=$install_dir; + + # Load the Environment variables from the app's service + local env_var=$(systemctl show $service.service -p "Environment" --value) + [ -n "$env_var" ] && export $env_var; + + # Force `php` to its intended version + # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` + local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + local phpflags=$(ynh_app_setting_get --app=$app --key=phpflags) + if [ -n "$phpversion" ] + then + eval "php() { php${phpversion} ${phpflags} \"\$@\"; }" + export -f php + fi + + # Source the EnvironmentFiles from the app's service + local env_files=($(systemctl show $service.service -p "EnvironmentFiles" --value)) + if [ ${#env_files[*]} -gt 0 ] + then + # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. + set -a + for file in ${env_files[*]} + do + [[ $file = /* ]] && source $file + done + set +a + fi + + # Activate the Python environment, if it exists + if [ -f $install_dir/venv/bin/activate ] + then + # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. + set -a + source $install_dir/venv/bin/activate + set +a + fi + + # cd into the WorkingDirectory set in the service, or default to the install_dir + local env_dir=$(systemctl show $service.service -p "WorkingDirectory" --value) + [ -z $env_dir ] && env_dir=$install_dir; + cd $env_dir + + # Spawn the app shell + su -s /bin/bash $app +} diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt new file mode 100644 index 000000000..d5a1f4335 --- /dev/null +++ b/helpers/helpers.v2.1.d/apt @@ -0,0 +1,636 @@ +#!/bin/bash + +# Check if apt is free to use, or wait, until timeout. +# +# [internal] +# +# usage: ynh_wait_dpkg_free +# | exit: Return 1 if dpkg is broken +# +# Requires YunoHost version 3.3.1 or higher. +ynh_wait_dpkg_free() { + local try + set +o xtrace # set +x + # With seq 1 17, timeout will be almost 30 minutes + for try in $(seq 1 17); do + # Check if /var/lib/dpkg/lock is used by another process + if lsof /var/lib/dpkg/lock >/dev/null; then + echo "apt is already in use..." + # Sleep an exponential time at each round + sleep $((try * try)) + else + # Check if dpkg hasn't been interrupted and is fully available. + # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + local dpkg_dir="/var/lib/dpkg/updates/" + + # For each file in $dpkg_dir + while read dpkg_file <&9; do + # Check if the name of this file contains only numbers. + if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then + # If so, that a remaining of dpkg. + ynh_print_err "dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + set -o xtrace # set -x + return 1 + fi + done 9<<<"$(ls -1 $dpkg_dir)" + set -o xtrace # set -x + return 0 + fi + done + echo "apt still used, but timeout reached !" + set -o xtrace # set -x +} + +# Check either a package is installed or not +# +# example: ynh_package_is_installed --package=yunohost && echo "installed" +# +# usage: ynh_package_is_installed --package=name +# | arg: -p, --package= - the package name to check +# | ret: 0 if the package is installed, 1 else. +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_is_installed() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=package=) + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + dpkg-query --show --showformat='${Status}' "$package" 2>/dev/null \ + | grep --count "ok installed" &>/dev/null +} + +# Get the version of an installed package +# +# example: version=$(ynh_package_version --package=yunohost) +# +# [internal] +# +# usage: ynh_package_version --package=name +# | arg: -p, --package= - the package name to get version +# | ret: the version or an empty string +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_version() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=package=) + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_package_is_installed "$package"; then + dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null + else + echo '' + fi +} + +# APT wrapper for non-interactive operation +# +# [internal] +# +# usage: ynh_apt update +# +# Requires YunoHost version 2.4.0.3 or higher. +ynh_apt() { + ynh_wait_dpkg_free + LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0 $@ +} + +# Update package index files +# +# [internal] +# +# usage: ynh_package_update +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_update() { + ynh_apt update +} + +# Install package(s) +# +# [internal] +# +# usage: ynh_package_install name [name [...]] +# | arg: name - the package name to install +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_install() { + ynh_apt --no-remove --option Dpkg::Options::=--force-confdef \ + --option Dpkg::Options::=--force-confold install $@ +} + +# Remove package(s) +# +# [internal] +# +# usage: ynh_package_remove name [name [...]] +# | arg: name - the package name to remove +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_remove() { + ynh_apt remove $@ +} + +# Remove package(s) and their uneeded dependencies +# +# [internal] +# +# usage: ynh_package_autoremove name [name [...]] +# | arg: name - the package name to remove +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_autoremove() { + ynh_apt autoremove $@ +} + +# Purge package(s) and their uneeded dependencies +# +# [internal] +# +# usage: ynh_package_autopurge name [name [...]] +# | arg: name - the package name to autoremove and purge +# +# Requires YunoHost version 2.7.2 or higher. +ynh_package_autopurge() { + ynh_apt autoremove --purge $@ +} + +# Build and install a package from an equivs control file +# +# [internal] +# +# example: generate an empty control file with `equivs-control`, adjust its +# content and use helper to build and install the package: +# ynh_package_install_from_equivs /path/to/controlfile +# +# usage: ynh_package_install_from_equivs controlfile +# | arg: controlfile - path of the equivs control file +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_install_from_equivs() { + local controlfile=$1 + + # retrieve package information + local pkgname=$(grep '^Package: ' $controlfile | cut --delimiter=' ' --fields=2) # Retrieve the name of the debian package + local pkgversion=$(grep '^Version: ' $controlfile | cut --delimiter=' ' --fields=2) # And its version number + [[ -z "$pkgname" || -z "$pkgversion" ]] \ + && ynh_die --message="Invalid control file" # Check if this 2 variables aren't empty. + + # Update packages cache + ynh_package_update + + # Build and install the package + local TMPDIR=$(mktemp --directory) + + # Make sure to delete the legacy compat file + # It's now handle somewhat magically through the control file + rm -f /usr/share/equivs/template/debian/compat + + # Note that the cd executes into a sub shell + # Create a fake deb package with equivs-build and the given control file + # Install the fake package without its dependencies with dpkg + # Install missing dependencies with ynh_package_install + ynh_wait_dpkg_free + cp "$controlfile" "${TMPDIR}/control" + ( + cd "$TMPDIR" + LC_ALL=C equivs-build ./control 2>&1 + LC_ALL=C dpkg --force-depends --install "./${pkgname}_${pkgversion}_all.deb" 2>&1 | tee ./dpkg_log + ) + + ynh_package_install --fix-broken \ + || { # If the installation failed + # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) + # Parse the list of problematic dependencies from dpkg's log ... + # (relevant lines look like: "foo-ynh-deps depends on bar; however:") + local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')" + # Fake an install of those dependencies to see the errors + # The sed command here is, Print only from 'Reading state info' to the end. + [[ -n "$problematic_dependencies" ]] && ynh_package_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 + ynh_die --message="Unable to install dependencies" + } + [[ -n "$TMPDIR" ]] && rm --recursive --force $TMPDIR # Remove the temp dir. + + # check if the package is actually installed + ynh_package_is_installed "$pkgname" +} + +YNH_INSTALL_APP_DEPENDENCIES_REPLACE="true" + +# Define and install dependencies with a equivs control file +# +# [packagingv1] +# +# This helper can/should only be called once per app +# +# example : ynh_install_app_dependencies dep1 dep2 "dep3|dep4|dep5" +# +# usage: ynh_install_app_dependencies dep [dep [...]] +# | arg: dep - the package name to install in dependence. +# | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc). +# +# Requires YunoHost version 2.6.4 or higher. +ynh_install_app_dependencies() { + local dependencies=$@ + # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) + dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" + local dependencies=${dependencies//|/ | } + + local version=$(ynh_read_manifest --manifest_key="version") + if [ -z "${version}" ] || [ "$version" == "null" ]; then + version="1.0" + fi + local dep_app=${app//_/-} # Replace all '_' by '-' + + # Handle specific versions + if [[ "$dependencies" =~ [\<=\>] ]]; then + # Replace version specifications by relationships syntax + # https://www.debian.org/doc/debian-policy/ch-relationships.html + # Sed clarification + # [^(\<=\>] ignore if it begins by ( or < = >. To not apply twice. + # [\<=\>] matches < = or > + # \+ matches one or more occurence of the previous characters, for >= or >>. + # [^,]\+ matches all characters except ',' + # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)' + dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')" + fi + + # Check for specific php dependencies which requires sury + # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni" + # The (?<=php) syntax corresponds to lookbehind ;) + local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u) + + if [[ -n "$specific_php_version" ]] + then + # Cover a small edge case where a packager could have specified "php7.4-pwet php5-gni" which is confusing + [[ $(echo $specific_php_version | wc -l) -eq 1 ]] \ + || ynh_die --message="Inconsistent php versions in dependencies ... found : $specific_php_version" + + dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common" + + local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + + # If the PHP version changed, remove the old fpm conf + if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$specific_php_version" ]; then + local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" + + if [[ -f "$old_php_finalphpconf" ]] + then + ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf" + ynh_remove_fpm_config + fi + fi + # Store phpversion into the config of this app + ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version + + # Set the default php version back as the default version for php-cli. + if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION + then + update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION + fi + elif grep --quiet 'php' <<< "$dependencies"; then + ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION + fi + + local psql_installed="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)" + + # The first time we run ynh_install_app_dependencies, we will replace the + # entire control file (This is in particular meant to cover the case of + # upgrade script where ynh_install_app_dependencies is called with this + # expected effect) Otherwise, any subsequent call will add dependencies + # to those already present in the equivs control file. + if [[ $YNH_INSTALL_APP_DEPENDENCIES_REPLACE == "true" ]] + then + YNH_INSTALL_APP_DEPENDENCIES_REPLACE="false" + else + local current_dependencies="" + if ynh_package_is_installed --package="${dep_app}-ynh-deps" + then + current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) " + current_dependencies=${current_dependencies// | /|} + fi + dependencies="$current_dependencies, $dependencies" + fi + + cat >/tmp/${dep_app}-ynh-deps.control </dev/null + then + ynh_package_autopurge ${dep_app}-ynh-deps + fi +} + +# Install packages from an extra repository properly. +# +# [packagingv1] +# +# usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" [--key=key_url] [--name=name] +# | arg: -r, --repo= - Complete url of the extra repository. +# | arg: -p, --package= - The packages to install from this extra repository +# | arg: -k, --key= - url to get the public key. +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_install_extra_app_dependencies() { + # Declare an array to define the options of this helper. + local legacy_args=rpkn + local -A args_array=([r]=repo= [p]=package= [k]=key= [n]=name=) + local repo + local package + local key + local name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + key=${key:-} + + # Set a key only if asked + if [ -n "$key" ]; then + key="--key=$key" + fi + # Add an extra repository for those packages + ynh_install_extra_repo --repo="$repo" $key --priority=995 --name=$name + + # Install requested dependencies from this extra repository. + ynh_install_app_dependencies "$package" + + # Force to upgrade to the last version... + # Without doing apt install, an already installed dep is not upgraded + local apps_auto_installed="$(apt-mark showauto $package)" + ynh_package_install "$package" + [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed + + # Remove this extra repository after packages are installed + ynh_remove_extra_repo --name=$name +} + +# Add an extra repository correctly, pin it and get the key. +# +# [internal] +# +# usage: ynh_install_extra_repo --repo="repo" [--key=key_url] [--priority=priority_value] [--name=name] [--append] +# | arg: -r, --repo= - Complete url of the extra repository. +# | arg: -k, --key= - url to get the public key. +# | arg: -p, --priority= - Priority for the pin +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_install_extra_repo() { + # Declare an array to define the options of this helper. + local legacy_args=rkpna + local -A args_array=([r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append) + local repo + local key + local priority + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + append=${append:-0} + key=${key:-} + priority=${priority:-} + + if [ $append -eq 1 ]; then + append="--append" + wget_append="tee --append" + else + append="" + wget_append="tee" + fi + + # Split the repository into uri, suite and components. + # Remove "deb " at the beginning of the repo. + repo="${repo#deb }" + + # Get the uri + local uri="$(echo "$repo" | awk '{ print $1 }')" + + # Get the suite + local suite="$(echo "$repo" | awk '{ print $2 }')" + + # Get the components + local component="${repo##$uri $suite }" + + # Add the repository into sources.list.d + ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append + + # Pin the new repo with the default priority, so it won't be used for upgrades. + # Build $pin from the uri without http and any sub path + local pin="${uri#*://}" + pin="${pin%%/*}" + # Set a priority only if asked + if [ -n "$priority" ]; then + priority="--priority=$priority" + fi + ynh_pin_repo --package="*" --pin="origin \"$pin\"" $priority --name="$name" $append + + # Get the public key for the repo + if [ -n "$key" ]; then + mkdir --parents "/etc/apt/trusted.gpg.d" + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg >/dev/null + fi + + # Update the list of package with the new repo + ynh_package_update +} + +# Remove an extra repository and the assiociated configuration. +# +# [internal] +# +# usage: ynh_remove_extra_repo [--name=name] +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_remove_extra_repo() { + # Declare an array to define the options of this helper. + local legacy_args=n + local -A args_array=([n]=name=) + local name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + + ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list" + # Sury pinning is managed by the regenconf in the core... + [[ "$name" == "extra_php_version" ]] || ynh_secure_remove "/etc/apt/preferences.d/$name" + if [ -e /etc/apt/trusted.gpg.d/$name.gpg ]; then + ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg" + fi + + # (Do we even create a .asc file anywhere ...?) + if [ -e /etc/apt/trusted.gpg.d/$name.asc ]; then + ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.asc" + fi + + # Update the list of package to exclude the old repo + ynh_package_update +} + +# Add a repository. +# +# [internal] +# +# usage: ynh_add_repo --uri=uri --suite=suite --component=component [--name=name] [--append] +# | arg: -u, --uri= - Uri of the repository. +# | arg: -s, --suite= - Suite of the repository. +# | arg: -c, --component= - Component of the repository. +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# Example for a repo like deb http://forge.yunohost.org/debian/ stretch stable +# uri suite component +# ynh_add_repo --uri=http://forge.yunohost.org/debian/ --suite=stretch --component=stable +# +# Requires YunoHost version 3.8.1 or higher. +ynh_add_repo() { + # Declare an array to define the options of this helper. + local legacy_args=uscna + local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append) + local uri + local suite + local component + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + append=${append:-0} + + if [ $append -eq 1 ]; then + append="tee --append" + else + append="tee" + fi + + mkdir --parents "/etc/apt/sources.list.d" + # Add the new repo in sources.list.d + echo "deb $uri $suite $component" \ + | $append "/etc/apt/sources.list.d/$name.list" +} + +# Pin a repository. +# +# [internal] +# +# usage: ynh_pin_repo --package=packages --pin=pin_filter [--priority=priority_value] [--name=name] [--append] +# | arg: -p, --package= - Packages concerned by the pin. Or all, *. +# | arg: -i, --pin= - Filter for the pin. +# | arg: -p, --priority= - Priority for the pin +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# See https://manpages.debian.org/stretch/apt/apt_preferences.5.en.html#How_APT_Interprets_Priorities for information about pinning. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_pin_repo() { + # Declare an array to define the options of this helper. + local legacy_args=pirna + local -A args_array=([p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append) + local package + local pin + local priority + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package="${package:-*}" + priority=${priority:-50} + name="${name:-$app}" + append=${append:-0} + + if [ $append -eq 1 ]; then + append="tee --append" + else + append="tee" + fi + + # Sury pinning is managed by the regenconf in the core... + [[ "$name" != "extra_php_version" ]] || return 0 + + mkdir --parents "/etc/apt/preferences.d" + echo "Package: $package +Pin: $pin +Pin-Priority: $priority +" \ + | $append "/etc/apt/preferences.d/$name" +} diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup new file mode 100644 index 000000000..a596ac9e0 --- /dev/null +++ b/helpers/helpers.v2.1.d/backup @@ -0,0 +1,507 @@ +#!/bin/bash + +CAN_BIND=${CAN_BIND:-1} + +# Add a file or a directory to the list of paths to backup +# +# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory] +# | arg: -s, --src_path= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. +# | arg: -d, --dest_path= - destination file or directory inside the backup dir +# | arg: -b, --is_big - Indicate data are big (mail, video, image ...) +# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it. +# +# This helper can be used both in a system backup hook, and in an app backup script +# +# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it +# creates the parent destination directory +# +# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`. +# +# Example in the context of a wordpress app : +# ``` +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" +# # => This line will be added into CSV file +# # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf" +# +# #Deprecated usages (maintained for retro-compatibility) +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" +# +# ``` +# +# How to use `--is_big`: +# +# `--is_big` is used to specify that this part of the backup can be quite huge. +# So, you don't want that your package does backup that part during ynh_backup_before_upgrade. +# In the same way, an user may doesn't want to backup this big part of the app for +# each of his backup. And so handle that part differently. +# +# As this part of your backup may not be done, your restore script has to handle it. +# In your restore script, use `--not_mandatory` with `ynh_restore_file` +# As well in your remove script, you should not remove those data ! Or an user may end up with +# a failed upgrade restoring an app without data anymore ! +# +# To have the benefit of `--is_big` while doing a backup, you can whether set the environement +# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect +# only that backup command. +# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect +# all backups for this app until the setting is removed. +# +# Requires YunoHost version 2.4.0 or higher. +# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory` +ynh_backup() { + # TODO find a way to avoid injection by file strange naming ! + + # Declare an array to define the options of this helper. + local legacy_args=sdbm + local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory) + local src_path + local dest_path + local is_big + local not_mandatory + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + dest_path="${dest_path:-}" + is_big="${is_big:-0}" + not_mandatory="${not_mandatory:-0}" + + BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} + test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data) + + # If backing up core only (used by ynh_backup_before_upgrade), + # don't backup big data items + if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then + if [ $BACKUP_CORE_ONLY -eq 1 ]; then + ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set." + else + ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set." + fi + return 0 + fi + + # ============================================================================== + # Format correctly source and destination paths + # ============================================================================== + # Be sure the source path is not empty + if [ ! -e "$src_path" ]; then + ynh_print_warn --message="Source path '${src_path}' does not exist" + if [ "$not_mandatory" == "0" ]; then + # This is a temporary fix for fail2ban config files missing after the migration to stretch. + if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then + touch "${src_path}" + ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!" + else + return 1 + fi + else + return 0 + fi + fi + + # Transform the source path as an absolute path + # If it's a dir remove the ending / + src_path=$(realpath "$src_path") + + # If there is no destination path, initialize it with the source path + # relative to "/". + # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost + if [[ -z "$dest_path" ]]; then + dest_path="${src_path#/}" + + else + if [[ "${dest_path:0:1}" == "/" ]]; then + + # If the destination path is an absolute path, transform it as a path + # relative to the current working directory ($YNH_CWD) + # + # If it's an app backup script that run this helper, YNH_CWD is equal to + # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/ + # + # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR + dest_path="${dest_path#$YNH_CWD/}" + + # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD + if [[ "${dest_path:0:1}" == "/" ]]; then + dest_path="${dest_path#/}" + fi + fi + + # Complete dest_path if ended by a / + if [[ "${dest_path: -1}" == "/" ]]; then + dest_path="${dest_path}/$(basename $src_path)" + fi + fi + + # Check if dest_path already exists in tmp archive + if [[ -e "${dest_path}" ]]; then + ynh_print_err --message="Destination path '${dest_path}' already exist" + return 1 + fi + + # Add the relative current working directory to the destination path + local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}" + rel_dir="${rel_dir%/}/" + dest_path="${rel_dir}${dest_path}" + dest_path="${dest_path#/}" + # ============================================================================== + + # ============================================================================== + # Write file to backup into backup_list + # ============================================================================== + local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g') + local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g') + echo "\"${src}\",\"${dest}\"" >>"${YNH_BACKUP_CSV}" + + # ============================================================================== + + # Create the parent dir of the destination path + # It's for retro compatibility, some script consider ynh_backup creates this dir + mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}") +} + +# Restore all files that were previously backuped in a core backup script or app backup script +# +# usage: ynh_restore +# +# Requires YunoHost version 2.6.4 or higher. +ynh_restore() { + # Deduce the relative path of $YNH_CWD + local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}" + REL_DIR="${REL_DIR%/}/" + + # For each destination path begining by $REL_DIR + cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \ + | while read line; do + local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)") + local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)") + ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH" + done +} + +# Return the path in the archive where has been stocked the origin path +# +# [internal] +# +# usage: _get_archive_path ORIGIN_PATH +_get_archive_path() { + # For security reasons we use csv python library to read the CSV + python3 -c " +import sys +import csv +with open(sys.argv[1], 'r') as backup_file: + backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest']) + for row in backup_csv: + if row['source']==sys.argv[2].strip('\"'): + print(row['dest']) + sys.exit(0) + raise Exception('Original path for %s not found' % sys.argv[2]) + " "${YNH_BACKUP_CSV}" "$1" + return $? +} + +# Restore a file or a directory +# +# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory] +# | arg: -o, --origin_path= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive +# | arg: -d, --dest_path= - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv` +# | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it. +# +# Use the registered path in backup_list by ynh_backup to restore the file at the right place. +# +# examples: +# ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf" +# # You can also use relative paths: +# ynh_restore_file -o "conf/nginx.conf" +# +# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in +# `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed. +# +# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into +# `/etc/nginx/conf.d/$domain.d/$app.conf` +# if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into +# `/etc/nginx/conf.d/$domain.d/$app.conf` +# +# Requires YunoHost version 2.6.4 or higher. +# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory +ynh_restore_file() { + # Declare an array to define the options of this helper. + local legacy_args=odm + local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory) + local origin_path + local dest_path + local not_mandatory + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + origin_path="/${origin_path#/}" + # Default value for dest_path = /$origin_path + dest_path="${dest_path:-$origin_path}" + not_mandatory="${not_mandatory:-0}" + + local archive_path="$YNH_CWD${origin_path}" + # If archive_path doesn't exist, search for a corresponding path in CSV + if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then + if [ "$not_mandatory" == "0" ]; then + archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")" + else + return 0 + fi + fi + + # Move the old directory if it already exists + if [[ -e "${dest_path}" ]]; then + # Check if the file/dir size is less than 500 Mo + if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then + local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')" + mkdir --parents "$(dirname "$backup_file")" + mv "${dest_path}" "$backup_file" # Move the current file or directory + else + ynh_secure_remove --file=${dest_path} + fi + fi + + # Restore origin_path into dest_path + mkdir --parents $(dirname "$dest_path") + + # Do a copy if it's just a mounting point + if mountpoint --quiet $YNH_BACKUP_DIR; then + if [[ -d "${archive_path}" ]]; then + archive_path="${archive_path}/." + mkdir --parents "$dest_path" + fi + cp --archive "$archive_path" "${dest_path}" + # Do a move if YNH_BACKUP_DIR is already a copy + else + mv "$archive_path" "${dest_path}" + fi + + # Boring hack for nginx conf file mapped to php7.3 + # Note that there's no need to patch the fpm config because most php apps + # will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch + if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}" + then + sed -i 's/php7.3/php7.4/g' "${dest_path}" + fi +} + +# Calculate and store a file checksum into the app settings +# +# usage: ynh_store_file_checksum --file=file +# | arg: -f, --file= - The file on which the checksum will performed, then stored. +# +# $app should be defined when calling this helper +# +# Requires YunoHost version 2.6.4 or higher. +ynh_store_file_checksum() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file= [u]=update_only) + local file + local update_only + update_only="${update_only:-0}" + + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + + # If update only, we don't save the new checksum if no old checksum exist + if [ $update_only -eq 1 ]; then + local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + if [ -z "${checksum_value}" ]; then + unset backup_file_checksum + return 0 + fi + fi + + ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) + + if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ... + local file_path_base64=$(echo "$file" | base64 -w0) + mkdir -p /var/cache/yunohost/appconfbackup/ + cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64} + fi + + # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup + if [ -n "${backup_file_checksum-}" ]; then + # Print the diff between the previous file and the new one. + # diff return 1 if the files are different, so the || true + diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true + fi + # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. + unset backup_file_checksum +} + +# Verify the checksum and backup the file if it's different +# +# usage: ynh_backup_if_checksum_is_different --file=file +# | arg: -f, --file= - The file on which the checksum test will be perfomed. +# | ret: the name of a backup file, or nothing +# +# This helper is primarily meant to allow to easily backup personalised/manually +# modified config files. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_backup_if_checksum_is_different() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum + backup_file_checksum="" + if [ -n "$checksum_value" ]; then # Proceed only if a value was stored into the app settings + if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different + + backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')" + mkdir --parents "$(dirname "$backup_file_checksum")" + cp --archive "$file" "$backup_file_checksum" # Backup the current file + ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + echo "$backup_file_checksum" # Return the name of the backup file + if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + local file_path_base64=$(echo "$file" | base64 -w0) + if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64} + then + ynh_print_warn "Diff with the original file:" + diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true + fi + fi + fi + fi +} + +# Delete a file checksum from the app settings +# +# usage: ynh_delete_file_checksum --file=file +# | arg: -f, --file= - The file for which the checksum will be deleted +# +# $app should be defined when calling this helper +# +# Requires YunoHost version 3.3.1 or higher. +ynh_delete_file_checksum() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + ynh_app_setting_delete --app=$app --key=$checksum_setting_name +} + +# Checks a backup archive exists +# +# [internal] +# +ynh_backup_archive_exists() { + yunohost backup list --output-as json --quiet \ + | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null +} + +# Make a backup in case of failed upgrade +# +# [packagingv1] +# +# usage: ynh_backup_before_upgrade +# +# Usage in a package script: +# ``` +# ynh_backup_before_upgrade +# ynh_clean_setup () { +# ynh_restore_upgradebackup +# } +# ynh_abort_if_errors +# ``` +# +# Requires YunoHost version 2.7.2 or higher. +ynh_backup_before_upgrade() { + if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]; then + ynh_print_warn --message="This app doesn't have any backup script." + return + fi + backup_number=1 + local old_backup_number=2 + local app_bck=${app//_/-} # Replace all '_' by '-' + NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} + + if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then + # Check if a backup already exists with the prefix 1 + if ynh_backup_archive_exists "$app_bck-pre-upgrade1"; then + # Prefix becomes 2 to preserve the previous backup + backup_number=2 + old_backup_number=1 + fi + + # Create backup + BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug + if [ "$?" -eq 0 ]; then + # If the backup succeeded, remove the previous backup + if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then + # Remove the previous backup only if it exists + yunohost backup delete $app_bck-pre-upgrade$old_backup_number >/dev/null + fi + else + ynh_die --message="Backup failed, the upgrade process was aborted." + fi + else + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" + fi +} + +# Restore a previous backup if the upgrade process failed +# +# [packagingv1] +# +# usage: ynh_restore_upgradebackup +# +# Usage in a package script: +# ``` +# ynh_backup_before_upgrade +# ynh_clean_setup () { +# ynh_restore_upgradebackup +# } +# ynh_abort_if_errors +# ``` +# +# Requires YunoHost version 2.7.2 or higher. +ynh_restore_upgradebackup() { + ynh_print_err --message="Upgrade failed." + local app_bck=${app//_/-} # Replace all '_' by '-' + + NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} + + if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then + # Check if an existing backup can be found before removing and restoring the application. + if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"; then + # Remove the application then restore it + yunohost app remove $app + # Restore the backup + yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug + if [[ -d /etc/yunohost/apps/$app ]] + then + ynh_die --message="The app was restored to the way it was before the failed upgrade." + else + ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|" + fi + fi + else + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" + fi +} diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config new file mode 100644 index 000000000..d2cc2760e --- /dev/null +++ b/helpers/helpers.v2.1.d/config @@ -0,0 +1,354 @@ +#!/bin/bash + +_ynh_app_config_get_one() { + local short_setting="$1" + local type="$2" + local bind="$3" + local getter="get__${short_setting}" + # Get value from getter if exists + if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + old[$short_setting]="$($getter)" + formats[${short_setting}]="yaml" + + elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + old[$short_setting]="$("get__${bind%%(*}" $short_setting $type $bind)" + formats[${short_setting}]="yaml" + + elif [[ "$bind" == "null" ]]; then + old[$short_setting]="YNH_NULL" + + # Get value from app settings or from another file + elif [[ "$type" == "file" ]]; then + if [[ "$bind" == "settings" ]]; then + ynh_die --message="File '${short_setting}' can't be stored in settings" + fi + old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" + file_hash[$short_setting]="true" + + # Get multiline text from settings or from a full file + elif [[ "$type" == "text" ]]; then + if [[ "$bind" == "settings" ]]; then + old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" + elif [[ "$bind" == *":"* ]]; then + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + else + old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" + fi + + # Get value from a kind of key/value file + else + local bind_after="" + if [[ "$bind" == "settings" ]]; then + bind=":/etc/yunohost/apps/$app/settings.yml" + fi + local bind_key_="$(echo "$bind" | cut -d: -f1)" + bind_key_=${bind_key_:-$short_setting} + if [[ "$bind_key_" == *">"* ]]; then + bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" + bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" + fi + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key_}" --after="${bind_after}")" + + fi +} +_ynh_app_config_apply_one() { + local short_setting="$1" + local setter="set__${short_setting}" + local bind="${binds[$short_setting]}" + local type="${types[$short_setting]}" + if [ "${changed[$short_setting]}" == "true" ]; then + # Apply setter if exists + if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + $setter + + elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + "set__${bind%%(*}" $short_setting $type $bind + + elif [[ "$bind" == "null" ]]; then + return + + # Save in a file + elif [[ "$type" == "file" ]]; then + if [[ "$bind" == "settings" ]]; then + ynh_die --message="File '${short_setting}' can't be stored in settings" + fi + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + if [[ "${!short_setting}" == "" ]]; then + ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_secure_remove --file="$bind_file" + ynh_delete_file_checksum --file="$bind_file" + ynh_print_info --message="File '$bind_file' removed" + else + ynh_backup_if_checksum_is_different --file="$bind_file" + if [[ "${!short_setting}" != "$bind_file" ]]; then + cp "${!short_setting}" "$bind_file" + fi + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info --message="File '$bind_file' overwritten with ${!short_setting}" + fi + + # Save value in app settings + elif [[ "$bind" == "settings" ]]; then + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_print_info --message="Configuration key '$short_setting' edited in app settings" + + # Save multiline text in a file + elif [[ "$type" == "text" ]]; then + if [[ "$bind" == *":"* ]]; then + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + fi + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + ynh_backup_if_checksum_is_different --file="$bind_file" + echo "${!short_setting}" >"$bind_file" + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info --message="File '$bind_file' overwritten with the content provided in question '${short_setting}'" + + # Set value into a kind of key/value file + else + local bind_after="" + local bind_key_="$(echo "$bind" | cut -d: -f1)" + if [[ "$bind_key_" == *">"* ]]; then + bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" + bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" + fi + bind_key_=${bind_key_:-$short_setting} + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + + ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}" + ynh_store_file_checksum --file="$bind_file" --update_only + + # We stored the info in settings in order to be able to upgrade the app + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_print_info --message="Configuration key '$bind_key_' edited into $bind_file" + + fi + fi +} +_ynh_app_config_get() { + # From settings + local lines + lines=$( + python3 <" in bind_section: + bind_section = bind_section + bind_panel_file + else: + bind_section = regex + bind_section + bind_panel_file + + for name, param in section.items(): + if not isinstance(param, dict): + continue + + bind = param.get('bind') + + if not bind: + if bind_section: + bind = bind_section + else: + bind = 'settings' + elif bind[-1] == ":" and bind_section and ":" in bind_section: + regex, bind_file = bind_section.split(":") + if ">" in bind: + bind = bind + bind_file + else: + bind = regex + bind + bind_file + if bind == "settings" and param.get('type', 'string') == 'file': + bind = 'null' + + print('|'.join([ + name, + param.get('type', 'string'), + bind + ])) +EOL + ) + for line in $lines; do + # Split line into short_setting, type and bind + IFS='|' read short_setting type bind <<<"$line" + binds[${short_setting}]="$bind" + types[${short_setting}]="$type" + file_hash[${short_setting}]="" + formats[${short_setting}]="" + ynh_app_config_get_one $short_setting $type $bind + done + +} + +_ynh_app_config_apply() { + for short_setting in "${!old[@]}"; do + ynh_app_config_apply_one $short_setting + done +} + +_ynh_app_config_show() { + for short_setting in "${!old[@]}"; do + if [[ "${old[$short_setting]}" != YNH_NULL ]]; then + if [[ "${formats[$short_setting]}" == "yaml" ]]; then + ynh_return "${short_setting}:" + ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/ /g')" + else + ynh_return "${short_setting}: '$(echo "${old[$short_setting]}" | sed "s/'/''/g" | sed ':a;N;$!ba;s/\n/\n\n/g')'" + fi + fi + done +} + +_ynh_app_config_validate() { + # Change detection + ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1 + local nothing_changed=true + local changes_validated=true + for short_setting in "${!old[@]}"; do + changed[$short_setting]=false + if [ -z ${!short_setting+x} ]; then + # Assign the var with the old value in order to allows multiple + # args validation + declare -g "$short_setting"="${old[$short_setting]}" + continue + fi + if [ ! -z "${file_hash[${short_setting}]}" ]; then + file_hash[old__$short_setting]="" + file_hash[new__$short_setting]="" + if [ -f "${old[$short_setting]}" ]; then + file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) + if [ -z "${!short_setting}" ]; then + changed[$short_setting]=true + nothing_changed=false + fi + fi + if [ -f "${!short_setting}" ]; then + file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1) + if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]; then + changed[$short_setting]=true + nothing_changed=false + fi + fi + else + if [[ "${!short_setting}" != "${old[$short_setting]}" ]]; then + changed[$short_setting]=true + nothing_changed=false + fi + fi + done + if [[ "$nothing_changed" == "true" ]]; then + ynh_print_info --message="Nothing has changed" + exit 0 + fi + + # Run validation if something is changed + ynh_script_progression --message="Validating the new configuration..." --weight=1 + + for short_setting in "${!old[@]}"; do + [[ "${changed[$short_setting]}" == "false" ]] && continue + local result="" + if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then + result="$(validate__$short_setting)" + elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + "validate__${bind%%(*}" $short_setting + fi + if [ -n "$result" ]; then + # + # Return a yaml such as: + # + # validation_errors: + # some_key: "An error message" + # some_other_key: "Another error message" + # + # We use changes_validated to know if this is + # the first validation error + if [[ "$changes_validated" == true ]]; then + ynh_return "validation_errors:" + fi + ynh_return " ${short_setting}: \"$result\"" + changes_validated=false + fi + done + + # If validation failed, exit the script right now (instead of going into apply) + # Yunohost core will pick up the errors returned via ynh_return previously + if [[ "$changes_validated" == "false" ]]; then + exit 0 + fi + +} + +ynh_app_config_get_one() { + _ynh_app_config_get_one $1 $2 $3 +} + +ynh_app_config_get() { + _ynh_app_config_get +} + +ynh_app_config_show() { + _ynh_app_config_show +} + +ynh_app_config_validate() { + _ynh_app_config_validate +} + +ynh_app_config_apply_one() { + _ynh_app_config_apply_one $1 +} +ynh_app_config_apply() { + _ynh_app_config_apply +} + +ynh_app_action_run() { + local runner="run__$1" + # Get value from getter if exists + if type -t "$runner" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + $runner + #ynh_return "result:" + #ynh_return "$(echo "${result}" | sed 's/^/ /g')" + else + ynh_die "No handler defined in app's script for action $1. If you are the maintainer of this app, you should define '$runner'" + fi +} + +ynh_app_config_run() { + declare -Ag old=() + declare -Ag changed=() + declare -Ag file_hash=() + declare -Ag binds=() + declare -Ag types=() + declare -Ag formats=() + + case $1 in + show) + ynh_app_config_get + ynh_app_config_show + ;; + apply) + max_progression=4 + ynh_script_progression --message="Reading config panel description and current configuration..." + ynh_app_config_get + + ynh_app_config_validate + + ynh_script_progression --message="Applying the new configuration..." + ynh_app_config_apply + ynh_script_progression --message="Configuration of $app completed" --last + ;; + *) + ynh_app_action_run $1 + esac +} diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban new file mode 100644 index 000000000..613dcc490 --- /dev/null +++ b/helpers/helpers.v2.1.d/fail2ban @@ -0,0 +1,136 @@ +#!/bin/bash + +# Create a dedicated fail2ban config (jail and filter conf files) +# +# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# +# usage 2: ynh_add_fail2ban_config --use_template +# | arg: -t, --use_template - Use this helper in template mode +# +# This will use a template in `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf` +# See the documentation of `ynh_add_config` for a description of the template +# format and how placeholders are replaced with actual variables. +# +# Generally your template will look like that by example (for synapse): +# ``` +# f2b_jail.conf: +# [__APP__] +# enabled = true +# port = http,https +# filter = __APP__ +# logpath = /var/log/__APP__/logfile.log +# maxretry = 3 +# ``` +# ``` +# f2b_filter.conf: +# [INCLUDES] +# before = common.conf +# [Definition] +# +# # Part of regex definition (just used to make more easy to make the global regex) +# __synapse_start_line = .? \- synapse\..+ \- +# +# # Regex definition. +# failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ +# +# ignoreregex = +# ``` +# +# ----------------------------------------------------------------------------- +# +# Note about the "failregex" option: +# +# regex to match the password failure messages in the logfile. The host must be +# matched by a group named "`host`". The tag "``" can be used for standard +# IP/hostname matching and is only an alias for `(?:::f{4,6}:)?(?P[\w\-.^_]+)` +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# To validate your regex you can test with this command: +# ``` +# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf +# ``` +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_fail2ban_config() { + # Declare an array to define the options of this helper. + local legacy_args=lrmptv + local -A args_array=([l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template) + local logpath + local failregex + local max_retry + local ports + local use_template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + max_retry=${max_retry:-3} + ports=${ports:-http,https} + use_template="${use_template:-0}" + + if [ "$use_template" -ne 1 ]; then + # Usage 1, no template. Build a config file from scratch. + test -n "$logpath" || ynh_die --message="ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die --message="ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + echo " +[__APP__] +enabled = true +port = __PORTS__ +filter = __APP__ +logpath = __LOGPATH__ +maxretry = __MAX_RETRY__ +" >"$YNH_APP_BASEDIR/conf/f2b_jail.conf" + + echo " +[INCLUDES] +before = common.conf +[Definition] +failregex = __FAILREGEX__ +ignoreregex = +" >"$YNH_APP_BASEDIR/conf/f2b_filter.conf" + fi + + ynh_add_config --template="f2b_jail.conf" --destination="/etc/fail2ban/jail.d/$app.conf" + ynh_add_config --template="f2b_filter.conf" --destination="/etc/fail2ban/filter.d/$app.conf" + + # if "$logpath" doesn't exist (as if using --use_template argument), assign + # "$logpath" using the one in the previously generated fail2ban conf file + if [ -z "${logpath:-}" ]; then + # the first sed deletes possibles spaces and the second one extract the path + logpath=$(grep "^logpath" "/etc/fail2ban/jail.d/$app.conf" | sed "s/ //g" | sed "s/logpath=//g") + fi + + # Create the folder and logfile if they doesn't exist, + # as fail2ban require an existing logfile before configuration + mkdir -p "/var/log/$app" + if [ ! -f "$logpath" ]; then + touch "$logpath" + fi + # Make sure log folder's permissions are correct + chown -R "$app:$app" "/var/log/$app" + chmod -R u=rwX,g=rX,o= "/var/log/$app" + + ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + + local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" + if [[ -n "$fail2ban_error" ]]; then + ynh_print_err --message="Fail2ban failed to load the jail for $app" + ynh_print_warn --message="${fail2ban_error#*WARNING}" + fi +} + +# Remove the dedicated fail2ban config (jail and filter conf files) +# +# usage: ynh_remove_fail2ban_config +# +# Requires YunoHost version 3.5.0 or higher. +ynh_remove_fail2ban_config() { + ynh_secure_remove --file="/etc/fail2ban/jail.d/$app.conf" + ynh_secure_remove --file="/etc/fail2ban/filter.d/$app.conf" + ynh_systemd_action --service_name=fail2ban --action=reload +} diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts new file mode 100644 index 000000000..f9ef5dc0b --- /dev/null +++ b/helpers/helpers.v2.1.d/getopts @@ -0,0 +1,215 @@ +#!/bin/bash + +# Internal helper design to allow helpers to use getopts to manage their arguments +# +# [internal] +# +# example: function my_helper() +# { +# local -A args_array=( [a]=arg1= [b]=arg2= [c]=arg3 ) +# local arg1 +# local arg2 +# local arg3 +# ynh_handle_getopts_args "$@" +# +# [...] +# } +# my_helper --arg1 "val1" -b val2 -c +# +# usage: ynh_handle_getopts_args "$@" +# | arg: $@ - Simply "$@" to tranfert all the positionnal arguments to the function +# +# This helper need an array, named "args_array" with all the arguments used by the helper +# that want to use ynh_handle_getopts_args +# Be carreful, this array has to be an associative array, as the following example: +# local -A args_array=( [a]=arg1 [b]=arg2= [c]=arg3 ) +# Let's explain this array: +# a, b and c are short options, -a, -b and -c +# arg1, arg2 and arg3 are the long options associated to the previous short ones. --arg1, --arg2 and --arg3 +# For each option, a short and long version has to be defined. +# Let's see something more significant +# local -A args_array=( [u]=user [f]=finalpath= [d]=database ) +# +# NB: Because we're using 'declare' without -g, the array will be declared as a local variable. +# +# Please keep in mind that the long option will be used as a variable to store the values for this option. +# For the previous example, that means that $finalpath will be fill with the value given as argument for this option. +# +# Also, in the previous example, finalpath has a '=' at the end. That means this option need a value. +# So, the helper has to be call with --finalpath /final/path, --finalpath=/final/path or -f /final/path, the variable $finalpath will get the value /final/path +# If there's many values for an option, -f /final /path, the value will be separated by a ';' $finalpath=/final;/path +# For an option without value, like --user in the example, the helper can be called only with --user or -u. $user will then get the value 1. +# +# To keep a retrocompatibility, a package can still call a helper, using getopts, with positional arguments. +# The "legacy mode" will manage the positional arguments and fill the variable in the same order than they are given in $args_array. +# e.g. for `my_helper "val1" val2`, arg1 will be filled with val1, and arg2 with val2. +# +# Requires YunoHost version 3.2.2 or higher. +ynh_handle_getopts_args() { + # Manage arguments only if there's some provided + set +o xtrace # set +x + if [ $# -ne 0 ]; then + # Store arguments in an array to keep each argument separated + local arguments=("$@") + + # For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u) + # And built parameters string for getopts + # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value) + local getopts_parameters="" + local option_flag="" + for option_flag in "${!args_array[@]}"; do + # Concatenate each option_flags of the array to build the string of arguments for getopts + # Will looks like 'abcd' for -a -b -c -d + # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob) + # Check the last character of the value associate to the option_flag + if [ "${args_array[$option_flag]: -1}" = "=" ]; then + # For an option with additionnal values, add a ':' after the letter for getopts. + getopts_parameters="${getopts_parameters}${option_flag}:" + else + getopts_parameters="${getopts_parameters}${option_flag}" + fi + # Check each argument given to the function + local arg="" + # ${#arguments[@]} is the size of the array + for arg in $(seq 0 $((${#arguments[@]} - 1))); do + # Escape options' values starting with -. Otherwise the - will be considered as another option. + arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}" + # And replace long option (value of the option_flag) by the short option, the option_flag itself + # (e.g. for [u]=user, --user will be -u) + # Replace long option with = (match the beginning of the argument) + arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")" + # And long option without = (match the whole line) + arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")" + done + done + + # Read and parse all the arguments + # Use a function here, to use standart arguments $@ and be able to use shift. + parse_arg() { + # Read all arguments, until no arguments are left + while [ $# -ne 0 ]; do + # Initialize the index of getopts + OPTIND=1 + # Parse with getopts only if the argument begin by -, that means the argument is an option + # getopts will fill $parameter with the letter of the option it has read. + local parameter="" + getopts ":$getopts_parameters" parameter || true + + if [ "$parameter" = "?" ]; then + ynh_die --message="Invalid argument: -${OPTARG:-}" + elif [ "$parameter" = ":" ]; then + ynh_die --message="-$OPTARG parameter requires an argument." + else + local shift_value=1 + # Use the long option, corresponding to the short option read by getopts, as a variable + # (e.g. for [u]=user, 'user' will be used as a variable) + # Also, remove '=' at the end of the long option + # The variable name will be stored in 'option_var' + local option_var="${args_array[$parameter]%=}" + # If this option doesn't take values + # if there's a '=' at the end of the long option name, this option takes values + if [ "${args_array[$parameter]: -1}" != "=" ]; then + # 'eval ${option_var}' will use the content of 'option_var' + eval ${option_var}=1 + else + # Read all other arguments to find multiple value for this option. + # Load args in a array + local all_args=("$@") + + # If the first argument is longer than 2 characters, + # There's a value attached to the option, in the same array cell + if [ ${#all_args[0]} -gt 2 ]; then + # Remove the option and the space, so keep only the value itself. + all_args[0]="${all_args[0]#-${parameter} }" + + # At this point, if all_args[0] start with "-", then the argument is not well formed + if [ "${all_args[0]:0:1}" == "-" ]; then + ynh_die --message="Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?" + fi + # Reduce the value of shift, because the option has been removed manually + shift_value=$((shift_value - 1)) + fi + + # Declare the content of option_var as a variable. + eval ${option_var}="" + # Then read the array value per value + local i + for i in $(seq 0 $((${#all_args[@]} - 1))); do + # If this argument is an option, end here. + if [ "${all_args[$i]:0:1}" == "-" ]; then + # Ignore the first value of the array, which is the option itself + if [ "$i" -ne 0 ]; then + break + fi + else + # Ignore empty parameters + if [ -n "${all_args[$i]}" ]; then + # Else, add this value to this option + # Each value will be separated by ';' + if [ -n "${!option_var}" ]; then + # If there's already another value for this option, add a ; before adding the new value + eval ${option_var}+="\;" + fi + + # Remove the \ that escape - at beginning of values. + all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}" + + # For the record. + # We're using eval here to get the content of the variable stored itself as simple text in $option_var... + # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var} + # But... ${!option_var} can't be used as left part of an assignation. + # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself. + # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one! + + eval ${option_var}+='"${all_args[$i]}"' + fi + shift_value=$((shift_value + 1)) + fi + done + fi + fi + + # Shift the parameter and its argument(s) + shift $shift_value + done + } + + # LEGACY MODE + # Check if there's getopts arguments + if [ "${arguments[0]:0:1}" != "-" ]; then + # If not, enter in legacy mode and manage the arguments as positionnal ones.. + # Dot not echo, to prevent to go through a helper output. But print only in the log. + set -x + echo "! Helper used in legacy mode !" >/dev/null + set +x + local i + for i in $(seq 0 $((${#arguments[@]} - 1))); do + # Try to use legacy_args as a list of option_flag of the array args_array + # Otherwise, fallback to getopts_parameters to get the option_flag. But an associative arrays isn't always sorted in the correct order... + # Remove all ':' in getopts_parameters + getopts_parameters=${legacy_args:-${getopts_parameters//:/}} + # Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument. + option_flag=${getopts_parameters:$i:1} + if [ -z "$option_flag" ]; then + ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored." + continue + fi + # Use the long option, corresponding to the option_flag, as a variable + # (e.g. for [u]=user, 'user' will be used as a variable) + # Also, remove '=' at the end of the long option + # The variable name will be stored in 'option_var' + local option_var="${args_array[$option_flag]%=}" + + # Store each value given as argument in the corresponding variable + # The values will be stored in the same order than $args_array + eval ${option_var}+='"${arguments[$i]}"' + done + unset legacy_args + else + # END LEGACY MODE + # Call parse_arg and pass the modified list of args as an array of arguments. + parse_arg "${arguments[@]}" + fi + fi + set -o xtrace # set -x +} diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go new file mode 100644 index 000000000..0e18301f7 --- /dev/null +++ b/helpers/helpers.v2.1.d/go @@ -0,0 +1,247 @@ +#!/bin/bash + +ynh_go_try_bash_extension() { + if [ -x src/configure ]; then + src/configure && make -C src || { + ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." + } + fi +} + +goenv_install_dir="/opt/goenv" +go_version_path="$goenv_install_dir/versions" +# goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable. +export GOENV_ROOT="$goenv_install_dir" + +# Load the version of Go for an app, and set variables. +# +# ynh_use_go has to be used in any app scripts before using Go for the first time. +# This helper will provide alias and variables to use in your scripts. +# +# To use gem or Go, use the alias `ynh_gem` and `ynh_go` +# Those alias will use the correct version installed for the app +# For example: use `ynh_gem install` instead of `gem install` +# +# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_go` +# And propagate $PATH to sudo with $ynh_go_load_path +# Exemple: `ynh_exec_as $app $ynh_go_load_path $ynh_gem install` +# +# $PATH contains the path of the requested version of Go. +# However, $PATH is duplicated into $go_path to outlast any manipulation of $PATH +# You can use the variable `$ynh_go_load_path` to quickly load your Go version +# in $PATH for an usage into a separate script. +# Exemple: `$ynh_go_load_path $install_dir/script_that_use_gem.sh` +# +# +# Finally, to start a Go service with the correct version, 2 solutions +# Either the app is dependent of Go or gem, but does not called it directly. +# In such situation, you need to load PATH +# `Environment="__YNH_GO_LOAD_PATH__"` +# `ExecStart=__INSTALL_DIR__/my_app` +# You will replace __YNH_GO_LOAD_PATH__ with $ynh_go_load_path +# +# Or Go start the app directly, then you don't need to load the PATH variable +# `ExecStart=__YNH_GO__ my_app run` +# You will replace __YNH_GO__ with $ynh_go +# +# +# one other variable is also available +# - $go_path: The absolute path to Go binaries for the chosen version. +# +# usage: ynh_use_go +# +# Requires YunoHost version 3.2.2 or higher. +ynh_use_go () { + go_version=$(ynh_app_setting_get --app=$app --key=go_version) + + # Get the absolute path of this version of Go + go_path="$go_version_path/$go_version/bin" + + # Allow alias to be used into bash script + shopt -s expand_aliases + + # Create an alias for the specific version of Go and a variable as fallback + ynh_go="$go_path/go" + alias ynh_go="$ynh_go" + + # Load the path of this version of Go in $PATH + if [[ :$PATH: != *":$go_path"* ]]; then + PATH="$go_path:$PATH" + fi + # Create an alias to easily load the PATH + ynh_go_load_path="PATH=$PATH" + + # Sets the local application-specific Go version + pushd $install_dir + $goenv_install_dir/bin/goenv local $go_version + popd +} + +# Install a specific version of Go +# +# ynh_install_go will install the version of Go provided as argument by using goenv. +# +# This helper creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv +# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) +# +# Don't forget to execute go-dependent command in a login environment +# (e.g. sudo --login option) +# When not possible (e.g. in systemd service definition), please use direct path +# to goenv shims (e.g. $goenv_ROOT/shims/bundle) +# +# usage: ynh_install_go --go_version=go_version +# | arg: -v, --go_version= - Version of go to install. +# +# Requires YunoHost version 3.2.2 or higher. +ynh_install_go () { + # Declare an array to define the options of this helper. + local legacy_args=v + local -A args_array=( [v]=go_version= ) + local go_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Load goenv path in PATH + local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Go prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Move an existing Go binary, to avoid to block goenv + test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv + + # Install or update goenv + goenv="$(command -v goenv $goenv_install_dir/bin/goenv | head -1)" + if [ -n "$goenv" ]; then + ynh_print_info --message="goenv already seems installed in \`$goenv'." + pushd "${goenv%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/syndbg/goenv.git"; then + echo "Trying to update with Git..." + git pull -q --tags origin master + cd .. + ynh_go_try_bash_extension + fi + popd + else + ynh_print_info --message="Installing goenv with Git..." + mkdir -p $goenv_install_dir + pushd $goenv_install_dir + git init -q + git remote add -f -t master origin https://github.com/syndbg/goenv.git > /dev/null 2>&1 + git checkout -q -b master origin/master + ynh_go_try_bash_extension + goenv=$goenv_install_dir/bin/goenv + popd + fi + + goenv_latest="$(command -v "$goenv_install_dir"/plugins/*/bin/goenv-latest goenv-latest | head -1)" + if [ -n "$goenv_latest" ]; then + ynh_print_info --message="\`goenv latest' command already available in \`$goenv_latest'." + pushd "${goenv_latest%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then + ynh_print_info --message="Trying to update xxenv-latest with git..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing xxenv-latest with Git..." + mkdir -p "${goenv_install_dir}/plugins" + git clone -q https://github.com/momo-lab/xxenv-latest.git "${goenv_install_dir}/plugins/xxenv-latest" + fi + + # Enable caching + mkdir -p "${goenv_install_dir}/cache" + + # Create shims directory if needed + mkdir -p "${goenv_install_dir}/shims" + + # Restore /usr/local/bin in PATH + PATH=$CLEAR_PATH + + # And replace the old Go binary + test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go + + # Install the requested version of Go + local final_go_version=$(goenv latest --print $go_version) + ynh_print_info --message="Installation of Go-$final_go_version" + goenv install --skip-existing $final_go_version + + # Store go_version into the config of this app + ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=go_version --value=$final_go_version + + # Cleanup Go versions + ynh_cleanup_go + + # Set environment for Go users + echo "#goenv +export GOENV_ROOT=$goenv_install_dir +export PATH=\"$goenv_install_dir/bin:$PATH\" +eval \"\$(goenv init -)\" +#goenv" > /etc/profile.d/goenv.sh + + # Load the environment + eval "$(goenv init -)" +} + +# Remove the version of Go used by the app. +# +# This helper will also cleanup Go versions +# +# usage: ynh_remove_go +ynh_remove_go () { + local go_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=go_version) + + # Load goenv path in PATH + local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Go prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Remove the line for this app + ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=go_version + + # Cleanup Go versions + ynh_cleanup_go +} + +# Remove no more needed versions of Go used by the app. +# +# This helper will check what Go version are no more required, +# and uninstall them +# If no app uses Go, goenv will be also removed. +# +# usage: ynh_cleanup_go +ynh_cleanup_go () { + + # List required Go versions + local installed_apps=$(yunohost app list --output-as json --quiet | jq -r .apps[].id) + local required_go_versions="" + for installed_app in $installed_apps + do + local installed_app_go_version=$(ynh_app_setting_get --app=$installed_app --key="go_version") + if [[ $installed_app_go_version ]] + then + required_go_versions="${installed_app_go_version}\n${required_go_versions}" + fi + done + + # Remove no more needed Go versions + local installed_go_versions=$(goenv versions --bare --skip-aliases | grep -Ev '/') + for installed_go_version in $installed_go_versions + do + if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1` + then + ynh_print_info --message="Removing of Go-$installed_go_version" + $goenv_install_dir/bin/goenv uninstall --force $installed_go_version + fi + done + + # If none Go version is required + if [[ ! $required_go_versions ]] + then + # Remove goenv environment configuration + ynh_print_info --message="Removing of goenv" + ynh_secure_remove --file="$goenv_install_dir" + ynh_secure_remove --file="/etc/profile.d/goenv.sh" + fi +} diff --git a/helpers/helpers.v2.1.d/hardware b/helpers/helpers.v2.1.d/hardware new file mode 100644 index 000000000..091f023f6 --- /dev/null +++ b/helpers/helpers.v2.1.d/hardware @@ -0,0 +1,105 @@ +#!/bin/bash + +# Get the total or free amount of RAM+swap on the system +# +# [packagingv1] +# +# usage: ynh_get_ram [--free|--total] [--ignore_swap|--only_swap] +# | arg: -f, --free - Count free RAM+swap +# | arg: -t, --total - Count total RAM+swap +# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM +# | arg: -o, --only_swap - Ignore real RAM, consider only swap +# | ret: the amount of free ram, in MB (MegaBytes) +# +# Requires YunoHost version 3.8.1 or higher. +ynh_get_ram() { + # Declare an array to define the options of this helper. + local legacy_args=ftso + local -A args_array=([f]=free [t]=total [s]=ignore_swap [o]=only_swap) + local free + local total + local ignore_swap + local only_swap + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + ignore_swap=${ignore_swap:-0} + only_swap=${only_swap:-0} + free=${free:-0} + total=${total:-0} + + if [ $free -eq $total ]; then + ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram" + ram=0 + # Use the total amount of ram + elif [ $free -eq 1 ]; then + local free_ram=$(LC_ALL=C vmstat --stats --unit M | grep "free memory" | awk '{print $1}') + local free_swap=$(LC_ALL=C vmstat --stats --unit M | grep "free swap" | awk '{print $1}') + local free_ram_swap=$((free_ram + free_swap)) + + # Use the total amount of free ram + local ram=$free_ram_swap + if [ $ignore_swap -eq 1 ]; then + # Use only the amount of free ram + ram=$free_ram + elif [ $only_swap -eq 1 ]; then + # Use only the amount of free swap + ram=$free_swap + fi + elif [ $total -eq 1 ]; then + local total_ram=$(LC_ALL=C vmstat --stats --unit M | grep "total memory" | awk '{print $1}') + local total_swap=$(LC_ALL=C vmstat --stats --unit M | grep "total swap" | awk '{print $1}') + local total_ram_swap=$((total_ram + total_swap)) + + local ram=$total_ram_swap + if [ $ignore_swap -eq 1 ]; then + # Use only the amount of free ram + ram=$total_ram + elif [ $only_swap -eq 1 ]; then + # Use only the amount of free swap + ram=$total_swap + fi + fi + + echo $ram +} + +# Return 0 or 1 depending if the system has a given amount of RAM+swap free or total +# +# [packagingv1] +# +# usage: ynh_require_ram --required=RAM [--free|--total] [--ignore_swap|--only_swap] +# | arg: -r, --required= - The amount to require, in MB +# | arg: -f, --free - Count free RAM+swap +# | arg: -t, --total - Count total RAM+swap +# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM +# | arg: -o, --only_swap - Ignore real RAM, consider only swap +# | ret: 1 if the ram is under the requirement, 0 otherwise. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_require_ram() { + # Declare an array to define the options of this helper. + local legacy_args=rftso + local -A args_array=([r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap) + local required + local free + local total + local ignore_swap + local only_swap + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Dunno if that's the right way to do, but that's some black magic to be able to + # forward the bool args to ynh_get_ram easily? + # If the variable $free is not empty, set it to '--free' + free=${free:+--free} + total=${total:+--total} + ignore_swap=${ignore_swap:+--ignore_swap} + only_swap=${only_swap:+--only_swap} + + local ram=$(ynh_get_ram $free $total $ignore_swap $only_swap) + + if [ $ram -lt $required ]; then + return 1 + else + return 0 + fi +} diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging new file mode 100644 index 000000000..accb8f9b0 --- /dev/null +++ b/helpers/helpers.v2.1.d/logging @@ -0,0 +1,347 @@ +#!/bin/bash + +# Print a message to stderr and exit +# +# usage: ynh_die --message=MSG [--ret_code=RETCODE] +# | arg: -m, --message= - Message to display +# | arg: -c, --ret_code= - Exit code to exit with +# +# Requires YunoHost version 2.4.0 or higher. +ynh_die() { + # Declare an array to define the options of this helper. + local legacy_args=mc + local -A args_array=([m]=message= [c]=ret_code=) + local message + local ret_code + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + ret_code=${ret_code:-1} + + echo "$message" 1>&2 + exit "$ret_code" +} + +# Display a message in the 'INFO' logging category +# +# usage: ynh_print_info --message="Some message" +# | arg: -m, --message= - Message to display +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_info() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=message=) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + echo "$message" >&$YNH_STDINFO +} + +# Main printer, just in case in the future we have to change anything about that. +# +# [internal] +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_log() { + echo -e "${1}" +} + +# Print a warning on stderr +# +# usage: ynh_print_warn --message="Text to print" +# | arg: -m, --message= - The text to print +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_warn() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=message=) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_print_log "${message}" >&2 +} + +# Print an error on stderr +# +# usage: ynh_print_err --message="Text to print" +# | arg: -m, --message= - The text to print +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_err() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=message=) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_print_log "[Error] ${message}" >&2 +} + +# Execute a command and print the result as an error +# +# usage: ynh_exec_err your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_err +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_err() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + ynh_print_err --message="$(eval $@)" + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + ynh_print_err --message="$("$@")" + fi +} + +# Execute a command and print the result as a warning +# +# usage: ynh_exec_warn your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_warn +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_warn() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + ynh_print_warn --message="$(eval $@)" + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + ynh_print_warn --message="$("$@")" + fi +} + +# Execute a command and force the result to be printed on stdout +# +# usage: ynh_exec_warn_less your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_warn +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_warn_less() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + eval $@ 2>&1 + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" 2>&1 + fi +} + +# Execute a command and redirect stdout in /dev/null +# +# usage: ynh_exec_quiet your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_warn +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_quiet() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + eval $@ > /dev/null + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" > /dev/null + fi +} + +# Execute a command and redirect stdout and stderr in /dev/null +# +# usage: ynh_exec_quiet your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_quiet +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_fully_quiet() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + eval $@ > /dev/null 2>&1 + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" > /dev/null 2>&1 + fi +} + +# Execute a command and redirect stderr in /dev/null. Print stderr on error. +# +# usage: ynh_exec_and_print_stderr_only_if_error your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_and_print_stderr_only_if_error +# +# Requires YunoHost version 11.2 or higher. +ynh_exec_and_print_stderr_only_if_error() { + logfile="$(mktemp)" + rc=0 + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" 2> "$logfile" || rc="$?" + if (( rc != 0 )); then + ynh_exec_warn cat "$logfile" + ynh_secure_remove "$logfile" + return "$rc" + fi +} + +# Remove any logs for all the following commands. +# +# usage: ynh_print_OFF +# +# [internal] +# +# WARNING: You should be careful with this helper, and never forget to use ynh_print_ON as soon as possible to restore the logging. +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_OFF() { + exec {BASH_XTRACEFD}>/dev/null +} + +# Restore the logging after ynh_print_OFF +# +# usage: ynh_print_ON +# +# [internal] +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_ON() { + exec {BASH_XTRACEFD}>&1 + # Print an echo only for the log, to be able to know that ynh_print_ON has been called. + echo ynh_print_ON >/dev/null +} + +# Initial definitions for ynh_script_progression +increment_progression=0 +previous_weight=0 +max_progression=-1 +# Set the scale of the progression bar +# progress_string(0,1,2) should have the size of the scale. +progress_scale=20 +progress_string2="####################" +progress_string1="++++++++++++++++++++" +progress_string0="...................." +# Define base_time when the file is sourced +base_time=$(date +%s) + +# Print a progress bar showing the progression of an app script +# +# usage: ynh_script_progression --message=message [--weight=weight] [--time] +# | arg: -m, --message= - The text to print +# | arg: -w, --weight= - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script. +# | arg: -t, --time - Print the execution time since the last call to this helper. Especially usefull to define weights. The execution time is given for the duration since the previous call. So the weight should be applied to this previous call. +# | arg: -l, --last - Use for the last call of the helper, to fill the progression bar. +# +# Requires YunoHost version 3.5.0 or higher. +ynh_script_progression() { + set +o xtrace # set +x + # Declare an array to define the options of this helper. + local legacy_args=mwtl + local -A args_array=([m]=message= [w]=weight= [t]=time [l]=last) + local message + local weight + local time + local last + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Re-disable xtrace, ynh_handle_getopts_args set it back + set +o xtrace # set +x + weight=${weight:-1} + + # Always activate time when running inside CI tests + if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + time=${time:-1} + else + time=${time:-0} + fi + + last=${last:-0} + + # Get execution time since the last $base_time + local exec_time=$(($(date +%s) - $base_time)) + base_time=$(date +%s) + + # Compute $max_progression (if we didn't already) + if [ "$max_progression" = -1 ]; then + # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented. + local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)" + # Get the number of call with a weight value + local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0) + + # Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight + local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]]*\).*/\1/g')" + # Get the weight of each occurrences of 'ynh_script_progression' in the script using -w + local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]]*\).*/\1/g')" + # Each value will be on a different line. + # Remove each 'end of line' and replace it by a '+' to sum the values. + local weight_values=$(($(echo "$weight_valuesA" "$weight_valuesB" | grep -v -E '^\s*$' | tr '\n' '+' | sed 's/+$/+0/g'))) + + # max_progression is a total number of calls to this helper. + # Less the number of calls with a weight value. + # Plus the total of weight values + max_progression=$(($helper_calls - $weight_calls + $weight_values)) + fi + + # Increment each execution of ynh_script_progression in this script by the weight of the previous call. + increment_progression=$(($increment_progression + $previous_weight)) + # Store the weight of the current call in $previous_weight for next call + previous_weight=$weight + + # Reduce $increment_progression to the size of the scale + if [ $last -eq 0 ]; then + local effective_progression=$(($increment_progression * $progress_scale / $max_progression)) + # If last is specified, fill immediately the progression_bar + else + local effective_progression=$progress_scale + fi + + # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task + # expected_progression is the progression expected after the current task + local expected_progression="$((($increment_progression + $weight) * $progress_scale / $max_progression - $effective_progression))" + if [ $last -eq 1 ]; then + expected_progression=0 + fi + # left_progression is the progression not yet done + local left_progression="$(($progress_scale - $effective_progression - $expected_progression))" + # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done. + local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}" + + local print_exec_time="" + if [ $time -eq 1 ] && [ "$exec_time" -gt 10 ]; then + print_exec_time=" [$(bc <<< "scale=1; $exec_time / 60" ) minutes]" + fi + + ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" + set -o xtrace # set -x +} + +# Return data to the YunoHost core for later processing +# (to be used by special hooks like app config panel and core diagnosis) +# +# usage: ynh_return somedata +# +# Requires YunoHost version 3.6.0 or higher. +ynh_return() { + echo "$1" >>"$YNH_STDRETURN" +} diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate new file mode 100644 index 000000000..efc1137c1 --- /dev/null +++ b/helpers/helpers.v2.1.d/logrotate @@ -0,0 +1,103 @@ +#!/bin/bash + +FIRST_CALL_TO_LOGROTATE="true" + +# Use logrotate to manage the logfile +# +# usage: ynh_use_logrotate [--logfile=/log/file] [--specific_user=user/group] +# | arg: -l, --logfile= - absolute path of logfile +# | arg: -u, --specific_user= - run logrotate as the specified user and group. If not specified logrotate is runned as root. +# +# If no `--logfile` is provided, `/var/log/$app` will be used as default. +# `logfile` can point to a directory or a file. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_use_logrotate() { + + # Stupid patch to ignore legacy --non-append and --nonappend + # which was never properly understood and improperly used and kind of bullshit + local all_args=( ${@} ) + for I in $(seq 0 $(($# - 1))) + do + if [[ "${all_args[$I]}" == "--non-append" ]] || [[ "${all_args[$I]}" == "--nonappend" ]] + then + unset all_args[$I] + fi + done + set -- "${all_args[@]}" + + # Argument parsing + local legacy_args=lu + local -A args_array=([l]=logfile= [u]=specific_user=) + local logfile + local specific_user + ynh_handle_getopts_args "$@" + logfile="${logfile:-}" + specific_user="${specific_user:-}" + + set -o noglob + if [[ -z "$logfile" ]]; then + logfile="/var/log/${app}/*.log" + elif [[ "${logfile##*.}" != "log" ]] && [[ "${logfile##*.}" != "txt" ]]; then + logfile="$logfile/*.log" + fi + set +o noglob + + for stuff in $logfile + do + mkdir --parents $(dirname "$stuff") + done + + local su_directive="" + if [[ -n "$specific_user" ]]; then + su_directive="su ${specific_user%/*} ${specific_user#*/}" + fi + + local tempconf="$(mktemp)" + cat << EOF >$tempconf +$logfile { + # Rotate if the logfile exceeds 100Mo + size 100M + # Keep 12 old log maximum + rotate 12 + # Compress the logs with gzip + compress + # Compress the log at the next cycle. So keep always 2 non compressed logs + delaycompress + # Copy and truncate the log to allow to continue write on it. Instead of moving the log. + copytruncate + # Do not trigger an error if the log is missing + missingok + # Do not rotate if the log is empty + notifempty + # Keep old logs in the same dir + noolddir + $su_directive +} +EOF + + if [[ "$FIRST_CALL_TO_LOGROTATE" == "true" ]] + then + cat $tempconf > /etc/logrotate.d/$app + else + cat $tempconf >> /etc/logrotate.d/$app + fi + + FIRST_CALL_TO_LOGROTATE="false" + + # Make sure permissions are correct (otherwise the config file could be ignored and the corresponding logs never rotated) + chmod 644 "/etc/logrotate.d/$app" + mkdir -p "/var/log/$app" + chmod 750 "/var/log/$app" +} + +# Remove the app's logrotate config. +# +# usage: ynh_remove_logrotate +# +# Requires YunoHost version 2.6.4 or higher. +ynh_remove_logrotate() { + if [ -e "/etc/logrotate.d/$app" ]; then + rm "/etc/logrotate.d/$app" + fi +} diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb new file mode 100644 index 000000000..d40d11bfe --- /dev/null +++ b/helpers/helpers.v2.1.d/mongodb @@ -0,0 +1,355 @@ +#!/bin/bash + +# Execute a mongo command +# +# example: ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("wekan")' +# example: ynh_mongo_exec --command="db.getMongo().getDBNames().indexOf(\"wekan\")" +# +# usage: ynh_mongo_exec [--user=user] [--password=password] [--authenticationdatabase=authenticationdatabase] [--database=database] [--host=host] [--port=port] --command="command" [--eval] +# | arg: -u, --user= - The user name to connect as +# | arg: -p, --password= - The user password +# | arg: -d, --authenticationdatabase= - The authenticationdatabase to connect to +# | arg: -d, --database= - The database to connect to +# | arg: -h, --host= - The host to connect to +# | arg: -P, --port= - The port to connect to +# | arg: -c, --command= - The command to evaluate +# | arg: -e, --eval - Evaluate instead of execute the command. +# +# +ynh_mongo_exec() { + # Declare an array to define the options of this helper. + local legacy_args=upadhPce + local -A args_array=( [u]=user= [p]=password= [a]=authenticationdatabase= [d]=database= [h]=host= [P]=port= [c]=command= [e]=eval ) + local user + local password + local authenticationdatabase + local database + local host + local port + local command + local eval + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + user="${user:-}" + password="${password:-}" + authenticationdatabase="${authenticationdatabase:-}" + database="${database:-}" + host="${host:-}" + port="${port:-}" + eval=${eval:-0} + + # If user is provided + if [ -n "$user" ] + then + user="--username=$user" + + # If password is provided + if [ -n "$password" ] + then + password="--password=$password" + fi + + # If authenticationdatabase is provided + if [ -n "$authenticationdatabase" ] + then + authenticationdatabase="--authenticationDatabase=$authenticationdatabase" + else + authenticationdatabase="--authenticationDatabase=admin" + fi + else + password="" + authenticationdatabase="" + fi + + # If host is provided + if [ -n "$host" ] + then + host="--host=$host" + fi + + # If port is provided + if [ -n "$port" ] + then + port="--port=$port" + fi + + # If eval is not provided + if [ $eval -eq 0 ] + then + # If database is provided + if [ -n "$database" ] + then + database="use $database" + else + database="" + fi + + mongosh --quiet --username $user --password $password --authenticationDatabase $authenticationdatabase --host $host --port $port < ./dump.bson +# +# usage: ynh_mongo_dump_db --database=database +# | arg: -d, --database= - The database name to dump +# | ret: the mongodump output +# +# +ynh_mongo_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=( [d]=database= ) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + mongodump --quiet --db="$database" --archive +} + +# Create a user +# +# [internal] +# +# usage: ynh_mongo_create_user --db_user=user --db_pwd=pwd --db_name=name +# | arg: -u, --db_user= - The user name to create +# | arg: -p, --db_pwd= - The password to identify user by +# | arg: -n, --db_name= - Name of the database to grant privilegies +# +# +ynh_mongo_create_user() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) + local db_user + local db_name + local db_pwd + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Create the user and set the user as admin of the db + ynh_mongo_exec --database="$db_name" --command='db.createUser( { user: "'${db_user}'", pwd: "'${db_pwd}'", roles: [ { role: "readWrite", db: "'${db_name}'" } ] } );' + + # Add clustermonitoring rights + ynh_mongo_exec --database="$db_name" --command='db.grantRolesToUser("'${db_user}'",[{ role: "clusterMonitor", db: "admin" }]);' +} + +# Check if a mongo database exists +# +# usage: ynh_mongo_database_exists --database=database +# | arg: -d, --database= - The database for which to check existence +# | exit: Return 1 if the database doesn't exist, 0 otherwise +# +# +ynh_mongo_database_exists() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [ $(ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("'${database}'")' --eval) -lt 0 ] + then + return 1 + else + return 0 + fi +} + +# Restore a database +# +# example: ynh_mongo_restore_db --database=wekan < ./dump.bson +# +# usage: ynh_mongo_restore_db --database=database +# | arg: -d, --database= - The database name to restore +# +# +ynh_mongo_restore_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=( [d]=database= ) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + mongorestore --quiet --db="$database" --archive +} + +# Drop a user +# +# [internal] +# +# usage: ynh_mongo_drop_user --db_user=user --db_name=name +# | arg: -u, --db_user= - The user to drop +# | arg: -n, --db_name= - Name of the database +# +# +ynh_mongo_drop_user() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -A args_array=( [u]=db_user= [n]=db_name= ) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_mongo_exec --database="$db_name" --command='db.dropUser("'$db_user'", {w: "majority", wtimeout: 5000})' +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# usage: ynh_mongo_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# +# After executing this helper, the password of the created database will be available in $db_pwd +# It will also be stored as "mongopwd" into the app settings. +# +# +ynh_mongo_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local new_db_pwd=$(ynh_string_random) # Generate a random password + # If $db_pwd is not provided, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + # Create the user and grant access to the database + ynh_mongo_create_user --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name" + + # Store the password in the app's config + ynh_app_setting_set --app=$app --key=db_pwd --value=$db_pwd +} + +# Remove a database if it exists, and the associated user +# +# usage: ynh_mongo_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# +# +ynh_mongo_remove_db() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -A args_array=( [u]=db_user= [n]=db_name= ) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists + ynh_mongo_drop_db --database=$db_name # Remove the database + else + ynh_print_warn --message="Database $db_name not found" + fi + + # Remove mongo user if it exists + ynh_mongo_drop_user --db_user=$db_user --db_name=$db_name +} + +# Install MongoDB and integrate MongoDB service in YunoHost +# +# usage: ynh_install_mongo [--mongo_version=mongo_version] +# | arg: -m, --mongo_version= - Version of MongoDB to install +# +# +ynh_install_mongo() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=mongo_version=) + local mongo_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + mongo_version="${mongo_version:-$YNH_MONGO_VERSION}" + + ynh_print_info --message="Installing MongoDB Community Edition ..." + local mongo_debian_release=$(ynh_get_debian_release) + + if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then + ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." + mongo_version="4.4" + fi + if [[ "$mongo_version" == "4.4" ]]; then + ynh_print_warn --message="Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release." + mongo_debian_release=buster + fi + + ynh_install_extra_app_dependencies --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc" + mongodb_servicename=mongod + + # Make sure MongoDB is started and enabled + systemctl enable $mongodb_servicename --quiet + systemctl daemon-reload --quiet + ynh_systemd_action --service_name=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" + + # Integrate MongoDB service in YunoHost + yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" + + # Store mongo_version into the config of this app + ynh_app_setting_set --app=$app --key=mongo_version --value=$mongo_version +} + +# Remove MongoDB +# Only remove the MongoDB service integration in YunoHost for now +# if MongoDB package as been removed +# +# usage: ynh_remove_mongo +# +# +ynh_remove_mongo() { + # Only remove the mongodb service if it is not installed. + if ! ynh_package_is_installed --package="mongodb*" + then + ynh_print_info --message="Removing MongoDB service..." + mongodb_servicename=mongod + # Remove the mongodb service + yunohost service remove $mongodb_servicename + ynh_secure_remove --file="/var/lib/mongodb" + ynh_secure_remove --file="/var/log/mongodb" + fi +} diff --git a/helpers/helpers.v2.1.d/multimedia b/helpers/helpers.v2.1.d/multimedia new file mode 100644 index 000000000..c860ae49f --- /dev/null +++ b/helpers/helpers.v2.1.d/multimedia @@ -0,0 +1,103 @@ +#!/bin/bash + +readonly MEDIA_GROUP=multimedia +readonly MEDIA_DIRECTORY=/home/yunohost.multimedia + +# Initialize the multimedia directory system +# +# usage: ynh_multimedia_build_main_dir +# +# Requires YunoHost version 4.2 or higher. +ynh_multimedia_build_main_dir() { + + ## Création du groupe multimedia + groupadd -f $MEDIA_GROUP + + ## Création des dossiers génériques + mkdir -p "$MEDIA_DIRECTORY" + mkdir -p "$MEDIA_DIRECTORY/share" + mkdir -p "$MEDIA_DIRECTORY/share/Music" + mkdir -p "$MEDIA_DIRECTORY/share/Picture" + mkdir -p "$MEDIA_DIRECTORY/share/Video" + mkdir -p "$MEDIA_DIRECTORY/share/eBook" + + ## Création des dossiers utilisateurs + for user in $(yunohost user list --output-as json | jq -r '.users | keys[]'); do + mkdir -p "$MEDIA_DIRECTORY/$user" + mkdir -p "$MEDIA_DIRECTORY/$user/Music" + mkdir -p "$MEDIA_DIRECTORY/$user/Picture" + mkdir -p "$MEDIA_DIRECTORY/$user/Video" + mkdir -p "$MEDIA_DIRECTORY/$user/eBook" + ln -sfn "$MEDIA_DIRECTORY/share" "$MEDIA_DIRECTORY/$user/Share" + # Création du lien symbolique dans le home de l'utilisateur. + #link will only be created if the home directory of the user exists and if it's located in '/home' folder + local user_home="$(getent passwd $user | cut -d: -f6 | grep '^/home/')" + if [[ -d "$user_home" ]]; then + ln -sfn "$MEDIA_DIRECTORY/$user" "$user_home/Multimedia" + fi + # Propriétaires des dossiers utilisateurs. + chown -R $user "$MEDIA_DIRECTORY/$user" + done + # Default yunohost hooks for post_user_create,delete will take care + # of creating/deleting corresponding multimedia folders when users + # are created/deleted in the future... + + ## Application des droits étendus sur le dossier multimedia. + # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other: + setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true + # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers. + setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true + # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl. + setfacl -RL -m m::rwx "$MEDIA_DIRECTORY" || true +} + +# Add a directory in yunohost.multimedia +# +# usage: ynh_multimedia_addfolder --source_dir="source_dir" --dest_dir="dest_dir" +# +# | arg: -s, --source_dir= - Source directory - The real directory which contains your medias. +# | arg: -d, --dest_dir= - Destination directory - The name and the place of the symbolic link, relative to "/home/yunohost.multimedia" +# +# This "directory" will be a symbolic link to a existing directory. +# +# Requires YunoHost version 4.2 or higher. +ynh_multimedia_addfolder() { + + # Declare an array to define the options of this helper. + local legacy_args=sd + local -A args_array=([s]=source_dir= [d]=dest_dir=) + local source_dir + local dest_dir + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Ajout d'un lien symbolique vers le dossier à partager + ln -sfn "$source_dir" "$MEDIA_DIRECTORY/$dest_dir" + + ## Application des droits étendus sur le dossier ajouté + # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other: + setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir" + # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers. + setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir" + # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl. + setfacl -RL -m m::rwx "$source_dir" +} + +# Allow an user to have an write authorisation in multimedia directories +# +# usage: ynh_multimedia_addaccess user_name +# +# | arg: -u, --user_name= - The name of the user which gain this access. +# +# Requires YunoHost version 4.2 or higher. +ynh_multimedia_addaccess() { + # Declare an array to define the options of this helper. + local legacy_args=u + declare -Ar args_array=([u]=user_name=) + local user_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + groupadd -f multimedia + usermod -a -G multimedia $user_name +} diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql new file mode 100644 index 000000000..c11f7989a --- /dev/null +++ b/helpers/helpers.v2.1.d/mysql @@ -0,0 +1,249 @@ +#!/bin/bash + +# Open a connection as a user +# +# usage: ynh_mysql_connect_as --user=user --password=password [--database=database] +# | arg: -u, --user= - the user name to connect as +# | arg: -p, --password= - the user password +# | arg: -d, --database= - the database to connect to +# +# examples: +# ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;" +# ynh_mysql_connect_as --user="user" --password="pass" < /path/to/file.sql +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_connect_as() { + # Declare an array to define the options of this helper. + local legacy_args=upd + local -A args_array=([u]=user= [p]=password= [d]=database=) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + mysql --user="$user" --password="$password" --batch "$database" +} + +# Execute a command as root user +# +# usage: ynh_mysql_execute_as_root --sql=sql [--database=database] +# | arg: -s, --sql= - the SQL command to execute +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_execute_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=sd + local -A args_array=([s]=sql= [d]=database=) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + mysql -B "$database" <<<"$sql" +} + +# Execute a command from a file as root user +# +# usage: ynh_mysql_execute_file_as_root --file=file [--database=database] +# | arg: -f, --file= - the file containing SQL commands +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_execute_file_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=fd + local -A args_array=([f]=file= [d]=database=) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + mysql -B "$database" <"$file" +} + +# Create a database and grant optionnaly privilegies to a user +# +# [internal] +# +# usage: ynh_mysql_create_db db [user [pwd]] +# | arg: db - the database name to create +# | arg: user - the user to grant privilegies +# | arg: pwd - the password to identify user by +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_create_db() { + local db=$1 + + local sql="CREATE DATABASE ${db};" + + # grant all privilegies to user + if [[ $# -gt 1 ]]; then + sql+=" GRANT ALL PRIVILEGES ON ${db}.* TO '${2}'@'localhost'" + if [[ -n ${3:-} ]]; then + sql+=" IDENTIFIED BY '${3}'" + fi + sql+=" WITH GRANT OPTION;" + fi + + ynh_mysql_execute_as_root --sql="$sql" +} + +# Drop a database +# +# [internal] +# +# If you intend to drop the database *and* the associated user, +# consider using ynh_mysql_remove_db instead. +# +# usage: ynh_mysql_drop_db db +# | arg: db - the database name to drop +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_drop_db() { + ynh_mysql_execute_as_root --sql="DROP DATABASE ${1};" +} + +# Dump a database +# +# usage: ynh_mysql_dump_db --database=database +# | arg: -d, --database= - the database name to dump +# | ret: The mysqldump output +# +# example: ynh_mysql_dump_db --database=roundcube > ./dump.sql +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + mysqldump --single-transaction --skip-dump-date --routines "$database" +} + +# Create a user +# +# [internal] +# +# usage: ynh_mysql_create_user user pwd [host] +# | arg: user - the user name to create +# | arg: pwd - the password to identify user by +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_create_user() { + ynh_mysql_execute_as_root \ + --sql="CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';" +} + +# Check if a mysql user exists +# +# [internal] +# +# usage: ynh_mysql_user_exists --user=user +# | arg: -u, --user= - the user for which to check existence +# | ret: 0 if the user exists, 1 otherwise. +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=user=) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]; then + return 1 + else + return 0 + fi +} + +# Drop a user +# +# [internal] +# +# usage: ynh_mysql_drop_user user +# | arg: user - the user name to drop +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_drop_user() { + ynh_mysql_execute_as_root --sql="DROP USER '${1}'@'localhost';" +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# [packagingv1] +# +# usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# +# After executing this helper, the password of the created database will be available in `$db_pwd` +# It will also be stored as "`mysqlpwd`" into the app settings. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_mysql_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Generate a random password + local new_db_pwd=$(ynh_string_random) + # If $db_pwd is not provided, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" + ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd +} + +# Remove a database if it exists, and the associated user +# +# [packagingv1] +# +# usage: ynh_mysql_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# +# Requires YunoHost version 2.6.4 or higher. +ynh_mysql_remove_db() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -Ar args_array=([u]=db_user= [n]=db_name=) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if mysqlshow | grep -q "^| $db_name "; then + ynh_mysql_drop_db $db_name + else + ynh_print_warn --message="Database $db_name not found" + fi + + # Remove mysql user if it exists + if ynh_mysql_user_exists --user=$db_user; then + ynh_mysql_drop_user $db_user + fi +} diff --git a/helpers/helpers.v2.1.d/network b/helpers/helpers.v2.1.d/network new file mode 100644 index 000000000..bed9dd402 --- /dev/null +++ b/helpers/helpers.v2.1.d/network @@ -0,0 +1,132 @@ +#!/bin/bash + +# Find a free port and return it +# +# [packagingv1] +# +# usage: ynh_find_port --port=begin_port +# | arg: -p, --port= - port to start to search +# | ret: the port number +# +# example: port=$(ynh_find_port --port=8080) +# +# Requires YunoHost version 2.6.4 or higher. +ynh_find_port() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=port=) + local port + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." + while ! ynh_port_available --port=$port; do + port=$((port + 1)) + done + echo $port +} + +# Test if a port is available +# +# [packagingv1] +# +# usage: ynh_find_port --port=XYZ +# | arg: -p, --port= - port to check +# | ret: 0 if the port is available, 1 if it is already used by another process. +# +# example: ynh_port_available --port=1234 || ynh_die --message="Port 1234 is needs to be available for this app" +# +# Requires YunoHost version 3.8.0 or higher. +ynh_port_available() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=port=) + local port + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Check if the port is free + if ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ":$port$"; then + return 1 + # This is to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up) + elif grep -q "port: '$port'" /etc/yunohost/apps/*/settings.yml; then + return 1 + else + return 0 + fi +} + +# Validate an IP address +# +# [internal] +# +# usage: ynh_validate_ip --family=family --ip_address=ip_address +# | ret: 0 for valid ip addresses, 1 otherwise +# +# example: ynh_validate_ip 4 111.222.333.444 +# +# Requires YunoHost version 2.2.4 or higher. +ynh_validate_ip() { + # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 + + # Declare an array to define the options of this helper. + local legacy_args=fi + local -A args_array=([f]=family= [i]=ip_address=) + local family + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 + + python3 /dev/stdin <"$n_install_dir/node_update.sh" <"/etc/cron.daily/node_update" <> $n_install_dir/node_update.log +EOF + + chmod +x "/etc/cron.daily/node_update" +} diff --git a/helpers/helpers.v2.1.d/permission b/helpers/helpers.v2.1.d/permission new file mode 100644 index 000000000..d3eb71c22 --- /dev/null +++ b/helpers/helpers.v2.1.d/permission @@ -0,0 +1,399 @@ +#!/bin/bash + +# Create a new permission for the app +# +# Example 1: `ynh_permission_create --permission=admin --url=/admin --additional_urls=domain.tld/admin /superadmin --allowed=alice bob \ +# --label="My app admin" --show_tile=true` +# +# This example will create a new permission permission with this following effect: +# - A tile named "My app admin" in the SSO will be available for the users alice and bob. This tile will point to the relative url '/admin'. +# - Only the user alice and bob will have the access to theses following url: /admin, domain.tld/admin, /superadmin +# +# +# Example 2: +# +# ynh_permission_create --permission=api --url=domain.tld/api --auth_header=false --allowed=visitors \ +# --label="MyApp API" --protected=true +# +# This example will create a new protected permission. So the admin won't be able to add/remove the visitors group of this permission. +# In case of an API with need to be always public it avoid that the admin break anything. +# With this permission all client will be allowed to access to the url 'domain.tld/api'. +# Note that in this case no tile will be show on the SSO. +# Note that the auth_header parameter is to 'false'. So no authentication header will be passed to the application. +# Generally the API is requested by an application and enabling the auth_header has no advantage and could bring some issues in some case. +# So in this case it's better to disable this option for all API. +# +# +# usage: ynh_permission_create --permission="permission" [--url="url"] [--additional_urls="second-url" [ "third-url" ]] [--auth_header=true|false] +# [--allowed=group1 [ group2 ]] [--label="label"] [--show_tile=true|false] +# [--protected=true|false] +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if 'show_tile' is enabled, this URL will be the URL of the tile. +# | arg: -A, --additional_urls= - (optional) List of additional URL for which access will be allowed/forbidden +# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true +# | arg: -a, --allowed= - (optional) A list of group/user to allow for the permission +# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. Default is "APP_LABEL (permission name)". +# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. Defaults to false for the permission different than 'main'. +# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. +# +# [packagingv1] +# +# If provided, 'url' or 'additional_urls' is assumed to be relative to the app domain/path if they +# start with '/'. For example: +# / -> domain.tld/app +# /admin -> domain.tld/app/admin +# domain.tld/app/api -> domain.tld/app/api +# +# 'url' or 'additional_urls' can be treated as a PCRE (not lua) regex if it starts with "re:". +# For example: +# re:/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$ +# re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$ +# +# Note that globally the parameter 'url' and 'additional_urls' are same. The only difference is: +# - 'url' is only one url, 'additional_urls' can be a list of urls. There are no limitation of 'additional_urls' +# - 'url' is used for the url of tile in the SSO (if enabled with the 'show_tile' parameter) +# +# +# About the authentication header (auth_header parameter). +# The SSO pass (by default) to the application theses following HTTP header (linked to the authenticated user) to the application: +# - "Auth-User": username +# - "Remote-User": username +# - "Email": user email +# +# Generally this feature is usefull to authenticate automatically the user in the application but in some case the application don't work with theses header and theses header need to be disabled to have the application to work correctly. +# See https://github.com/YunoHost/issues/issues/1420 for more informations +# +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_create() { + # Declare an array to define the options of this helper. + local legacy_args=puAhaltP + local -A args_array=([p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected=) + local permission + local url + local additional_urls + local auth_header + local allowed + local label + local show_tile + local protected + ynh_handle_getopts_args "$@" + url=${url:-} + additional_urls=${additional_urls:-} + auth_header=${auth_header:-} + allowed=${allowed:-} + label=${label:-} + show_tile=${show_tile:-} + protected=${protected:-} + + if [[ -n $url ]]; then + url=",url='$url'" + fi + + if [[ -n $additional_urls ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # By example: + # --additional_urls /urlA /urlB + # will be: + # additional_urls=['/urlA', '/urlB'] + additional_urls=",additional_urls=['${additional_urls//;/\',\'}']" + fi + + if [[ -n $auth_header ]]; then + if [ $auth_header == "true" ]; then + auth_header=",auth_header=True" + else + auth_header=",auth_header=False" + fi + fi + + if [[ -n $allowed ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # By example: + # --allowed alice bob + # will be: + # allowed=['alice', 'bob'] + allowed=",allowed=['${allowed//;/\',\'}']" + fi + + if [[ -n ${label:-} ]]; then + label=",label='$label'" + else + label=",label='$permission'" + fi + + if [[ -n ${show_tile:-} ]]; then + if [ $show_tile == "true" ]; then + show_tile=",show_tile=True" + else + show_tile=",show_tile=False" + fi + fi + + if [[ -n ${protected:-} ]]; then + if [ $protected == "true" ]; then + protected=",protected=True" + else + protected=",protected=False" + fi + fi + + yunohost tools shell -c "from yunohost.permission import permission_create; permission_create('$app.$permission' $url $additional_urls $auth_header $allowed $label $show_tile $protected)" +} + +# Remove a permission for the app (note that when the app is removed all permission is automatically removed) +# +# [packagingv1] +# +# example: ynh_permission_delete --permission=editors +# +# usage: ynh_permission_delete --permission="permission" +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_delete() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=permission=) + local permission + ynh_handle_getopts_args "$@" + + yunohost tools shell -c "from yunohost.permission import permission_delete; permission_delete('$app.$permission')" +} + +# Check if a permission exists +# +# [packagingv1] +# +# usage: ynh_permission_exists --permission=permission +# | arg: -p, --permission= - the permission to check +# | exit: Return 1 if the permission doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_exists() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=permission=) + local permission + ynh_handle_getopts_args "$@" + + yunohost user permission list "$app" --output-as json --quiet \ + | jq -e --arg perm "$app.$permission" '.permissions[$perm]' >/dev/null +} + +# Redefine the url associated to a permission +# +# [packagingv1] +# +# usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]] +# [--auth_header=true|false] [--clear_urls] +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) +# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if you want to remove url you can pass an empty sting as arguments (""). +# | arg: -a, --add_url= - (optional) List of additional url to add for which access will be allowed/forbidden. +# | arg: -r, --remove_url= - (optional) List of additional url to remove for which access will be allowed/forbidden +# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application +# | arg: -c, --clear_urls - (optional) Clean all urls (url and additional_urls) +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_url() { + # Declare an array to define the options of this helper. + local legacy_args=puarhc + local -A args_array=([p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls) + local permission + local url + local add_url + local remove_url + local auth_header + local clear_urls + ynh_handle_getopts_args "$@" + url=${url:-} + add_url=${add_url:-} + remove_url=${remove_url:-} + auth_header=${auth_header:-} + clear_urls=${clear_urls:-} + + if [[ -n $url ]]; then + url=",url='$url'" + fi + + if [[ -n $add_url ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --add_url /urlA /urlB + # will be: + # add_url=['/urlA', '/urlB'] + add_url=",add_url=['${add_url//;/\',\'}']" + fi + + if [[ -n $remove_url ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --remove_url /urlA /urlB + # will be: + # remove_url=['/urlA', '/urlB'] + remove_url=",remove_url=['${remove_url//;/\',\'}']" + fi + + if [[ -n $auth_header ]]; then + if [ $auth_header == "true" ]; then + auth_header=",auth_header=True" + else + auth_header=",auth_header=False" + fi + fi + + if [[ -n $clear_urls ]] && [ $clear_urls -eq 1 ]; then + clear_urls=",clear_urls=True" + fi + + yunohost tools shell -c "from yunohost.permission import permission_url; permission_url('$app.$permission' $url $add_url $remove_url $auth_header $clear_urls)" +} + +# Update a permission for the app +# +# [packagingv1] +# +# usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]] +# [--label="label"] [--show_tile=true|false] [--protected=true|false] +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: -a, --add= - the list of group or users to enable add to the permission +# | arg: -r, --remove= - the list of group or users to remove from the permission +# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. +# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO +# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_update() { + # Declare an array to define the options of this helper. + local legacy_args=parltP + local -A args_array=([p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected=) + local permission + local add + local remove + local label + local show_tile + local protected + ynh_handle_getopts_args "$@" + add=${add:-} + remove=${remove:-} + label=${label:-} + show_tile=${show_tile:-} + protected=${protected:-} + + if [[ -n $add ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --add alice bob + # will be: + # add=['alice', 'bob'] + add=",add=['${add//';'/"','"}']" + fi + if [[ -n $remove ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --remove alice bob + # will be: + # remove=['alice', 'bob'] + remove=",remove=['${remove//';'/"','"}']" + fi + + if [[ -n $label ]]; then + label=",label='$label'" + fi + + if [[ -n $show_tile ]]; then + if [ $show_tile == "true" ]; then + show_tile=",show_tile=True" + else + show_tile=",show_tile=False" + fi + fi + + if [[ -n $protected ]]; then + if [ $protected == "true" ]; then + protected=",protected=True" + else + protected=",protected=False" + fi + fi + + yunohost tools shell -c "from yunohost.permission import user_permission_update; user_permission_update('$app.$permission' $add $remove $label $show_tile $protected , force=True)" +} + +# Check if a permission has an user +# +# example: ynh_permission_has_user --permission=main --user=visitors +# +# usage: ynh_permission_has_user --permission=permission --user=user +# | arg: -p, --permission= - the permission to check +# | arg: -u, --user= - the user seek in the permission +# | exit: Return 1 if the permission doesn't have that user or doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.7.1 or higher. +ynh_permission_has_user() { + local legacy_args=pu + # Declare an array to define the options of this helper. + local -A args_array=([p]=permission= [u]=user=) + local permission + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! ynh_permission_exists --permission=$permission; then + return 1 + fi + + # Check both allowed and corresponding_users sections in the json + for section in "allowed" "corresponding_users"; do + if yunohost user permission info "$app.$permission" --output-as json --quiet \ + | jq -e --arg user $user --arg section $section '.[$section] | index($user)' >/dev/null; then + return 0 + fi + done + + return 1 +} + +# Check if a legacy permissions exist +# +# [packagingv1] +# +# usage: ynh_legacy_permissions_exists +# | exit: Return 1 if the permission doesn't exist, 0 otherwise +# +# Requires YunoHost version 4.1.2 or higher. +ynh_legacy_permissions_exists() { + for permission in "skipped" "unprotected" "protected"; do + if ynh_permission_exists --permission="legacy_${permission}_uris"; then + return 0 + fi + done + return 1 +} + +# Remove all legacy permissions +# +# [packagingv1] +# +# usage: ynh_legacy_permissions_delete_all +# +# example: +# if ynh_legacy_permissions_exists +# then +# ynh_legacy_permissions_delete_all +# # You can recreate the required permissions here with ynh_permission_create +# fi +# Requires YunoHost version 4.1.2 or higher. +ynh_legacy_permissions_delete_all() { + for permission in "skipped" "unprotected" "protected"; do + if ynh_permission_exists --permission="legacy_${permission}_uris"; then + ynh_permission_delete --permission="legacy_${permission}_uris" + fi + done +} diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php new file mode 100644 index 000000000..7fbe3f1ba --- /dev/null +++ b/helpers/helpers.v2.1.d/php @@ -0,0 +1,583 @@ +#!/bin/bash + +readonly YNH_DEFAULT_PHP_VERSION=7.4 +# Declare the actual PHP version to use. +# A packager willing to use another version of PHP can override the variable into its _common.sh. +YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} + +# Create a dedicated PHP-FPM config +# +# usage: ynh_add_fpm_config +# +# Case 1 (recommended) : your provided a snippet conf/extra_php-fpm.conf +# +# The actual PHP configuration will be automatically generated, +# and your extra_php-fpm.conf will be appended (typically contains PHP upload limits) +# +# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf +# +# Performance-related options in the PHP conf, such as : +# pm.max_children, pm.start_servers, pm.min_spare_servers pm.max_spare_servers +# are computed from two parameters called "usage" and "footprint" which can be set to low/medium/high. (cf details below) +# +# If you wish to tweak those, please initialize the settings `fpm_usage` and `fpm_footprint` +# *prior* to calling this helper. Otherwise, "low" will be used as a default for both values. +# +# Otherwise, if you want the user to have control over these, we encourage to create a config panel +# (which should ultimately be standardized by the core ...) +# +# Case 2 (deprecate) : you provided an entire conf/php-fpm.conf +# +# The configuration will be hydrated, replacing __FOOBAR__ placeholders with $foobar values, etc. +# +# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf +# +# ---------------------- +# +# fpm_footprint: Memory footprint of the service (low/medium/high). +# low - Less than 20 MB of RAM by pool. +# medium - Between 20 MB and 40 MB of RAM by pool. +# high - More than 40 MB of RAM by pool. +# N - Or you can specify a quantitative footprint as MB by pool (use watch -n0.5 ps -o user,cmd,%cpu,rss -u APP) +# +# fpm_usage: Expected usage of the service (low/medium/high). +# low - Personal usage, behind the SSO. +# medium - Low usage, few people or/and publicly accessible. +# high - High usage, frequently visited website. +# +# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM. +# So it will be used to defined 'pm.max_children' +# A lower value for the footprint will allow more children for 'pm.max_children'. And so for +# 'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the +# value of 'pm.max_children' +# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores. +# +# The usage value will defined the way php will handle the children for the pool. +# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the +# service is used, otherwise no child will stay alive. This config gives the lower footprint when the +# service is idle. But will use more proc since it has to start a child as soon it's used. +# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children +# equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request. +# The number of children can grow if needed. The footprint can stay low if the service is idle, but +# not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few +# children already available. +# Set as 'high', the process manager will be set at 'static'. There will be always as many children as +# 'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum +# RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many +# children ready to answer. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_fpm_config() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vufpdg + local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service [g]=group=) + local group + local phpversion + local usage + local footprint + local package + local dedicated_service + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package=${package:-} + group=${group:-} + + # The default behaviour is to use the template. + local autogenconf=false + usage="${usage:-}" + footprint="${footprint:-}" + if [ -n "$usage" ] || [ -n "$footprint" ] || [[ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]]; then + autogenconf=true + + # If no usage provided, default to the value existing in setting ... or to low + local fpm_usage_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_usage) + if [ -z "$usage" ] + then + usage=${fpm_usage_in_setting:-low} + ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage + fi + + # If no footprint provided, default to the value existing in setting ... or to low + local fpm_footprint_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_footprint) + if [ -z "$footprint" ] + then + footprint=${fpm_footprint_in_setting:-low} + ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint + fi + + fi + # Do not use a dedicated service by default + dedicated_service=${dedicated_service:-0} + + # Set the default PHP-FPM version by default + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + + # If the PHP version changed, remove the old fpm conf + # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper) + if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then + local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" + + if [[ -f "$old_php_finalphpconf" ]] + then + ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf" + ynh_remove_fpm_config + fi + fi + + # Legacy args (packager should just list their php dependency as regular apt dependencies... + if [ -n "$package" ]; then + # Install the additionnal packages from the default repository + ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future" + ynh_install_app_dependencies "$package" + fi + + if [ $dedicated_service -eq 1 ]; then + ynh_print_warn --message "Argument --dedicated_service of ynh_add_fpm_config is deprecated and to be removed in the future" + local fpm_service="${app}-phpfpm" + local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" + else + local fpm_service="php${phpversion}-fpm" + local fpm_config_dir="/etc/php/$phpversion/fpm" + fi + + # Create the directory for FPM pools + mkdir --parents "$fpm_config_dir/pool.d" + + ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" + ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" + ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service" + ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion + + # Migrate from mutual PHP service to dedicated one. + if [ $dedicated_service -eq 1 ]; then + local old_fpm_config_dir="/etc/php/$phpversion/fpm" + # If a config file exist in the common pool, move it. + if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]; then + ynh_print_info --message="Migrate to a dedicated php-fpm service for $app." + # Create a backup of the old file before migration + ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf" + # Remove the old PHP config file + ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf" + # Reload PHP to release the socket and allow the dedicated service to use it + ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload + fi + fi + + if [ $autogenconf == "false" ]; then + # Usage 1, use the template in conf/php-fpm.conf + local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" + # Make sure now that the template indeed exists + [ -e "$phpfpm_path" ] || ynh_die --message="Unable to find template to configure PHP-FPM." + else + # Usage 2, generate a PHP-FPM config file with ynh_get_scalable_phpfpm + + # Define the values to use for the configuration of PHP. + ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint + + local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app") + local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" + echo " +[__APP__] + +user = __APP__ +group = __PHPFPM_GROUP__ + +chdir = __INSTALL_DIR__ + +listen = /var/run/php/php__PHPVERSION__-fpm-__APP__.sock +listen.owner = www-data +listen.group = www-data + +pm = __PHP_PM__ +pm.max_children = __PHP_MAX_CHILDREN__ +pm.max_requests = 500 +request_terminate_timeout = 1d +" >"$phpfpm_path" + + if [ "$php_pm" = "dynamic" ]; then + echo " +pm.start_servers = __PHP_START_SERVERS__ +pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__ +pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__ +" >>"$phpfpm_path" + + elif [ "$php_pm" = "ondemand" ]; then + echo " +pm.process_idle_timeout = 10s +" >>"$phpfpm_path" + fi + + # Concatene the extra config. + if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then + cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path" + fi + fi + + local finalphpconf="$fpm_config_dir/pool.d/$app.conf" + ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" + + if [ -e "$YNH_APP_BASEDIR/conf/php-fpm.ini" ]; then + ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." + ynh_add_config --template="php-fpm.ini" --destination="$fpm_config_dir/conf.d/20-$app.ini" + fi + + if [ $dedicated_service -eq 1 ]; then + # Create a dedicated php-fpm.conf for the service + local globalphpconf=$fpm_config_dir/php-fpm-$app.conf + + echo "[global] +pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid +error_log = /var/log/php/fpm-php.__APP__.log +syslog.ident = php-fpm-__APP__ +include = __FINALPHPCONF__ +" >$YNH_APP_BASEDIR/conf/php-fpm-$app.conf + + ynh_add_config --template="php-fpm-$app.conf" --destination="$globalphpconf" + + # Create a config for a dedicated PHP-FPM service for the app + echo "[Unit] +Description=PHP __PHPVERSION__ FastCGI Process Manager for __APP__ +After=network.target + +[Service] +Type=notify +PIDFile=/run/php/php__PHPVERSION__-fpm-__APP__.pid +ExecStart=/usr/sbin/php-fpm__PHPVERSION__ --nodaemonize --fpm-config __GLOBALPHPCONF__ +ExecReload=/bin/kill -USR2 \$MAINPID + +[Install] +WantedBy=multi-user.target +" >$YNH_APP_BASEDIR/conf/$fpm_service + + # Create this dedicated PHP-FPM service + ynh_add_systemd_config --service=$fpm_service --template=$fpm_service + # Integrate the service in YunoHost admin panel + yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --description "Php-fpm dedicated to $app" + # Configure log rotate + ynh_use_logrotate --logfile=/var/log/php + # Restart the service, as this service is either stopped or only for this app + ynh_systemd_action --service_name=$fpm_service --action=restart + else + # Validate that the new php conf doesn't break php-fpm entirely + if ! php-fpm${phpversion} --test 2>/dev/null; then + php-fpm${phpversion} --test || true + ynh_secure_remove --file="$finalphpconf" + ynh_die --message="The new configuration broke php-fpm?" + fi + ynh_systemd_action --service_name=$fpm_service --action=reload + fi +} + +# Remove the dedicated PHP-FPM config +# +# usage: ynh_remove_fpm_config +# +# Requires YunoHost version 2.7.2 or higher. +ynh_remove_fpm_config() { + local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) + local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) + dedicated_service=${dedicated_service:-0} + # Get the version of PHP used by this app + local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + + # Assume default PHP-FPM version by default + phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}" + + # Assume default PHP files if not set + if [ -z "$fpm_config_dir" ]; then + fpm_config_dir="/etc/php/$YNH_DEFAULT_PHP_VERSION/fpm" + fpm_service="php$YNH_DEFAULT_PHP_VERSION-fpm" + fi + + ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" + if [ -e $fpm_config_dir/conf.d/20-$app.ini ]; then + ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" + fi + + if [ $dedicated_service -eq 1 ]; then + # Remove the dedicated service PHP-FPM service for the app + ynh_remove_systemd_config --service=$fpm_service + # Remove the global PHP-FPM conf + ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" + # Remove the service from the list of services known by YunoHost + yunohost service remove $fpm_service + elif ynh_package_is_installed --package="php${phpversion}-fpm"; then + ynh_systemd_action --service_name=$fpm_service --action=reload + fi + + # If the PHP version used is not the default version for YunoHost + # The second part with YNH_APP_PURGE is an ugly hack to guess that we're inside the remove script + # (we don't actually care about its value, we just check its not empty hence it exists) + if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + # Remove app dependencies ... but ideally should happen via an explicit call from packager + ynh_remove_app_dependencies + fi +} + +# Install another version of PHP. +# +# [internal] +# +# Legacy, to be remove on bullseye +# +# usage: ynh_install_php --phpversion=phpversion [--package=packages] +# | arg: -v, --phpversion= - Version of PHP to install. +# | arg: -p, --package= - Additionnal PHP packages to install +# +# Requires YunoHost version 3.8.1 or higher. +ynh_install_php() { + # Declare an array to define the options of this helper. + local legacy_args=vp + local -A args_array=([v]=phpversion= [p]=package=) + local phpversion + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package=${package:-} + + if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]; then + ynh_die --message="Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION" + fi + + ynh_install_app_dependencies "$package" +} + +# Remove the specific version of PHP used by the app. +# +# [internal] +# +# Legacy, to be remove on bullseye +# +# usage: ynh_remove_php +# +# Requires YunoHost version 3.8.1 or higher. +ynh_remove_php () { + ynh_remove_app_dependencies +} + +# Define the values to configure PHP-FPM +# +# [internal] +# +# usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print] +# | arg: -f, --footprint= - Memory footprint of the service (low/medium/high). +# low - Less than 20 MB of RAM by pool. +# medium - Between 20 MB and 40 MB of RAM by pool. +# high - More than 40 MB of RAM by pool. +# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value. +# To have this value, use the following command and stress the service. +# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP +# +# | arg: -u, --usage= - Expected usage of the service (low/medium/high). +# low - Personal usage, behind the SSO. +# medium - Low usage, few people or/and publicly accessible. +# high - High usage, frequently visited website. +# +# | arg: -p, --print - Print the result (intended for debug purpose only when packaging the app) +ynh_get_scalable_phpfpm() { + local legacy_args=ufp + # Declare an array to define the options of this helper. + local -A args_array=([u]=usage= [f]=footprint= [p]=print) + local usage + local footprint + local print + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Set all characters as lowercase + footprint=${footprint,,} + usage=${usage,,} + print=${print:-0} + + if [ "$footprint" = "low" ]; then + footprint=20 + elif [ "$footprint" = "medium" ]; then + footprint=35 + elif [ "$footprint" = "high" ]; then + footprint=50 + fi + + # Define the factor to determine min_spare_servers + # to avoid having too few children ready to start for heavy apps + if [ $footprint -le 20 ]; then + min_spare_servers_factor=8 + elif [ $footprint -le 35 ]; then + min_spare_servers_factor=5 + else + min_spare_servers_factor=3 + fi + + # Define the way the process manager handle child processes. + if [ "$usage" = "low" ]; then + php_pm=ondemand + elif [ "$usage" = "medium" ]; then + php_pm=dynamic + elif [ "$usage" = "high" ]; then + php_pm=static + else + ynh_die --message="Does not recognize '$usage' as an usage value." + fi + + # Get the total of RAM available, except swap. + local max_ram=$(ynh_get_ram --total --ignore_swap) + + at_least_one() { + # Do not allow value below 1 + if [ $1 -le 0 ]; then + echo 1 + else + echo $1 + fi + } + + # Define pm.max_children + # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. + # So if PHP-FPM start the maximum of children, it won't exceed half of the ram. + php_max_children=$(($max_ram / 2 / $footprint)) + # If process manager is set as static, use half less children. + # Used as static, there's always as many children as the value of pm.max_children + if [ "$php_pm" = "static" ]; then + php_max_children=$(($php_max_children / 2)) + fi + php_max_children=$(at_least_one $php_max_children) + + # To not overload the proc, limit the number of children to 4 times the number of cores. + local core_number=$(nproc) + local max_proc=$(($core_number * 4)) + if [ $php_max_children -gt $max_proc ]; then + php_max_children=$max_proc + fi + + # Get a potential forced value for php_max_children + local php_forced_max_children=$(ynh_app_setting_get --app=$app --key=php_forced_max_children) + if [ -n "$php_forced_max_children" ]; then + php_max_children=$php_forced_max_children + fi + + if [ "$php_pm" = "dynamic" ]; then + # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager + php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor)) + php_min_spare_servers=$(at_least_one $php_min_spare_servers) + + php_max_spare_servers=$(($php_max_children / 2)) + php_max_spare_servers=$(at_least_one $php_max_spare_servers) + + php_start_servers=$(($php_min_spare_servers + ($php_max_spare_servers - $php_min_spare_servers) / 2)) + php_start_servers=$(at_least_one $php_start_servers) + else + php_min_spare_servers=0 + php_max_spare_servers=0 + php_start_servers=0 + fi + + if [ $print -eq 1 ]; then + ynh_print_warn --message="Footprint=${footprint}Mb by pool." + ynh_print_warn --message="Process manager=$php_pm" + ynh_print_warn --message="Max RAM=${max_ram}Mb" + if [ "$php_pm" != "static" ]; then + ynh_print_warn --message="\nMax estimated footprint=$(($php_max_children * $footprint))" + ynh_print_warn --message="Min estimated footprint=$(($php_min_spare_servers * $footprint))" + fi + if [ "$php_pm" = "dynamic" ]; then + ynh_print_warn --message="Estimated average footprint=$(($php_max_spare_servers * $footprint))" + elif [ "$php_pm" = "static" ]; then + ynh_print_warn --message="Estimated footprint=$(($php_max_children * $footprint))" + fi + ynh_print_warn --message="\nRaw php-fpm values:" + ynh_print_warn --message="pm.max_children = $php_max_children" + if [ "$php_pm" = "dynamic" ]; then + ynh_print_warn --message="pm.start_servers = $php_start_servers" + ynh_print_warn --message="pm.min_spare_servers = $php_min_spare_servers" + ynh_print_warn --message="pm.max_spare_servers = $php_max_spare_servers" + fi + fi +} + +readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17 +# Declare the actual composer version to use. +# A packager willing to use another version of composer can override the variable into its _common.sh. +YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} + +# Execute a command with Composer +# +# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$install_dir] --commands="commands" +# | arg: -v, --phpversion - PHP version to use with composer +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path +# | arg: -c, --commands - Commands to execute. +# +# Requires YunoHost version 4.2 or higher. +ynh_composer_exec() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vwc + declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=) + local phpversion + local workdir + local commands + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + workdir="${workdir:-${install_dir:-$final_path}}" + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ + php${phpversion} "$workdir/composer.phar" $commands \ + -d "$workdir" --no-interaction --no-ansi 2>&1 +} + +# Install and initialize Composer in the given directory +# +# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$install_dir] [--install_args="--optimize-autoloader"] [--composerversion=composerversion] +# | arg: -v, --phpversion - PHP version to use with composer +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. +# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include +# | arg: -c, --composerversion - Composer version to install +# +# Requires YunoHost version 4.2 or higher. +ynh_install_composer() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vwac + declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) + local phpversion + local workdir + local install_args + local composerversion + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + workdir="${workdir:-$final_path}" + else + workdir="${workdir:-$install_dir}" + fi + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + install_args="${install_args:-}" + composerversion="${composerversion:-$YNH_COMPOSER_VERSION}" + + curl -sS https://getcomposer.org/installer \ + | COMPOSER_HOME="$workdir/.composer" \ + php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \ + || ynh_die --message="Unable to install Composer." + + # install dependencies + ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \ + || ynh_die --message="Unable to install core dependencies with Composer." +} diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql new file mode 100644 index 000000000..35b95cd5f --- /dev/null +++ b/helpers/helpers.v2.1.d/postgresql @@ -0,0 +1,310 @@ +#!/bin/bash + +PSQL_ROOT_PWD_FILE=/etc/yunohost/psql +PSQL_VERSION=13 + +# Open a connection as a user +# +# usage: ynh_psql_connect_as --user=user --password=password [--database=database] +# | arg: -u, --user= - the user name to connect as +# | arg: -p, --password= - the user password +# | arg: -d, --database= - the database to connect to +# +# examples: +# ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" +# ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_connect_as() { + # Declare an array to define the options of this helper. + local legacy_args=upd + local -A args_array=([u]=user= [p]=password= [d]=database=) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" +} + +# Execute a command as root user +# +# usage: ynh_psql_execute_as_root --sql=sql [--database=database] +# | arg: -s, --sql= - the SQL command to execute +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_execute_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=sd + local -A args_array=([s]=sql= [d]=database=) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ + $database <<<"$sql" +} + +# Execute a command from a file as root user +# +# usage: ynh_psql_execute_file_as_root --file=file [--database=database] +# | arg: -f, --file= - the file containing SQL commands +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_execute_file_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=fd + local -A args_array=([f]=file= [d]=database=) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ + $database <"$file" +} + +# Create a database and grant optionnaly privilegies to a user +# +# [internal] +# +# usage: ynh_psql_create_db db [user] +# | arg: db - the database name to create +# | arg: user - the user to grant privilegies +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_create_db() { + local db=$1 + local user=${2:-} + + local sql="CREATE DATABASE ${db};" + + # grant all privilegies to user + if [ -n "$user" ]; then + sql+="ALTER DATABASE ${db} OWNER TO ${user};" + sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" + fi + + ynh_psql_execute_as_root --sql="$sql" +} + +# Drop a database +# +# [internal] +# +# If you intend to drop the database *and* the associated user, +# consider using ynh_psql_remove_db instead. +# +# usage: ynh_psql_drop_db db +# | arg: db - the database name to drop +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_drop_db() { + local db=$1 + # First, force disconnection of all clients connected to the database + # https://stackoverflow.com/questions/17449420/postgresql-unable-to-drop-database-because-of-some-auto-connections-to-db + ynh_psql_execute_as_root --sql="REVOKE CONNECT ON DATABASE $db FROM public;" --database="$db" + ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db' AND pid <> pg_backend_pid();" --database="$db" + sudo --login --user=postgres dropdb $db +} + +# Dump a database +# +# usage: ynh_psql_dump_db --database=database +# | arg: -d, --database= - the database name to dump +# | ret: the psqldump output +# +# example: ynh_psql_dump_db 'roundcube' > ./dump.sql +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo --login --user=postgres pg_dump "$database" +} + +# Create a user +# +# [internal] +# +# usage: ynh_psql_create_user user pwd +# | arg: user - the user name to create +# | arg: pwd - the password to identify user by +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_create_user() { + local user=$1 + local pwd=$2 + ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" +} + +# Check if a psql user exists +# +# [packagingv1] +# +# usage: ynh_psql_user_exists --user=user +# | arg: -u, --user= - the user for which to check existence +# | exit: Return 1 if the user doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=user=) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user"; then + return 1 + else + return 0 + fi +} + +# Check if a psql database exists +# +# usage: ynh_psql_database_exists --database=database +# | arg: -d, --database= - the database for which to check existence +# | exit: Return 1 if the database doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_database_exists() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # if psql is not there, we cannot check the db + # though it could exists. + if ! command -v psql + then + ynh_print_err -m "PostgreSQL is not installed, impossible to check for db existence." + return 1 + elif ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then + return 1 + else + return 0 + fi +} + +# Drop a user +# +# [internal] +# +# usage: ynh_psql_drop_user user +# | arg: user - the user name to drop +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_drop_user() { + ynh_psql_execute_as_root --sql="DROP USER ${1};" +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# [packagingv1] +# +# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# +# After executing this helper, the password of the created database will be available in $db_pwd +# It will also be stored as "psqlpwd" into the app settings. +# +# Requires YunoHost version 2.7.13 or higher. +ynh_psql_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! ynh_psql_user_exists --user=$db_user; then + local new_db_pwd=$(ynh_string_random) # Generate a random password + # If $db_pwd is not provided, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + ynh_psql_create_user "$db_user" "$db_pwd" + elif [ -z $db_pwd ]; then + ynh_die --message="The user $db_user exists, please provide his password" + fi + + ynh_psql_create_db "$db_name" "$db_user" # Create the database + ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config +} + +# Remove a database if it exists, and the associated user +# +# [packagingv1] +# +# usage: ynh_psql_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# +# Requires YunoHost version 2.7.13 or higher. +ynh_psql_remove_db() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -A args_array=([u]=db_user= [n]=db_name=) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_psql_database_exists --database=$db_name; then # Check if the database exists + ynh_psql_drop_db $db_name # Remove the database + else + ynh_print_warn --message="Database $db_name not found" + fi + + # Remove psql user if it exists + if ynh_psql_user_exists --user=$db_user; then + ynh_psql_drop_user $db_user + else + ynh_print_warn --message="User $db_user not found" + fi +} + +# Create a master password and set up global settings +# +# [internal] +# +# usage: ynh_psql_test_if_first_run +# +# It also make sure that postgresql is installed and running +# Please always call this script in install and restore scripts +# +# Requires YunoHost version 2.7.13 or higher. +ynh_psql_test_if_first_run() { + + # Make sure postgresql is indeed installed + dpkg --list | grep -q "ii postgresql-$PSQL_VERSION" || ynh_die --message="postgresql-$PSQL_VERSION is not installed !?" + + yunohost tools regen-conf postgresql +} diff --git a/helpers/helpers.v2.1.d/redis b/helpers/helpers.v2.1.d/redis new file mode 100644 index 000000000..545bb8705 --- /dev/null +++ b/helpers/helpers.v2.1.d/redis @@ -0,0 +1,39 @@ +#!/bin/bash + +# get the first available redis database +# +# usage: ynh_redis_get_free_db +# | returns: the database number to use +ynh_redis_get_free_db() { + local result max db + result=$(redis-cli INFO keyspace) + + # get the num + max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+") + + db=0 + # default Debian setting is 15 databases + for i in $(seq 0 "$max") + do + if ! echo "$result" | grep -q "db$i" + then + db=$i + break 1 + fi + db=-1 + done + + test "$db" -eq -1 && ynh_die --message="No available Redis databases..." + + echo "$db" +} + +# Create a master password and set up global settings +# Please always call this script in install and restore scripts +# +# usage: ynh_redis_remove_db database +# | arg: database - the database to erase +ynh_redis_remove_db() { + local db=$1 + redis-cli -n "$db" flushdb +} diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby new file mode 100644 index 000000000..82a946935 --- /dev/null +++ b/helpers/helpers.v2.1.d/ruby @@ -0,0 +1,306 @@ +#!/bin/bash + +rbenv_install_dir="/opt/rbenv" +ruby_version_path="$rbenv_install_dir/versions" + +# RBENV_ROOT is the directory of rbenv, it needs to be loaded as a environment variable. +export RBENV_ROOT="$rbenv_install_dir" +export rbenv_root="$rbenv_install_dir" + +if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + build_ruby_dependencies="libjemalloc-dev curl build-essential libreadline-dev zlib1g-dev libsqlite3-dev libssl-dev libxml2-dev libxslt-dev autoconf automake bison libtool" + build_pkg_dependencies="${build_pkg_dependencies:-} $build_ruby_dependencies" +fi + +# Load the version of Ruby for an app, and set variables. +# +# ynh_use_ruby has to be used in any app scripts before using Ruby for the first time. +# This helper will provide alias and variables to use in your scripts. +# +# To use gem or Ruby, use the alias `ynh_gem` and `ynh_ruby` +# Those alias will use the correct version installed for the app +# For example: use `ynh_gem install` instead of `gem install` +# +# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_ruby` +# And propagate $PATH to sudo with $ynh_ruby_load_path +# Exemple: `ynh_exec_as $app $ynh_ruby_load_path $ynh_gem install` +# +# $PATH contains the path of the requested version of Ruby. +# However, $PATH is duplicated into $ruby_path to outlast any manipulation of $PATH +# You can use the variable `$ynh_ruby_load_path` to quickly load your Ruby version +# in $PATH for an usage into a separate script. +# Exemple: $ynh_ruby_load_path $final_path/script_that_use_gem.sh` +# +# +# Finally, to start a Ruby service with the correct version, 2 solutions +# Either the app is dependent of Ruby or gem, but does not called it directly. +# In such situation, you need to load PATH +# `Environment="__YNH_RUBY_LOAD_PATH__"` +# `ExecStart=__FINALPATH__/my_app` +# You will replace __YNH_RUBY_LOAD_PATH__ with $ynh_ruby_load_path +# +# Or Ruby start the app directly, then you don't need to load the PATH variable +# `ExecStart=__YNH_RUBY__ my_app run` +# You will replace __YNH_RUBY__ with $ynh_ruby +# +# +# one other variable is also available +# - $ruby_path: The absolute path to Ruby binaries for the chosen version. +# +# usage: ynh_use_ruby +# +# Requires YunoHost version 3.2.2 or higher. +ynh_use_ruby () { + ruby_version=$(ynh_app_setting_get --app=$app --key=ruby_version) + + # Get the absolute path of this version of Ruby + ruby_path="$ruby_version_path/$YNH_APP_INSTANCE_NAME/bin" + + # Allow alias to be used into bash script + shopt -s expand_aliases + + # Create an alias for the specific version of Ruby and a variable as fallback + ynh_ruby="$ruby_path/ruby" + alias ynh_ruby="$ynh_ruby" + # And gem + ynh_gem="$ruby_path/gem" + alias ynh_gem="$ynh_gem" + + # Load the path of this version of Ruby in $PATH + if [[ :$PATH: != *":$ruby_path"* ]]; then + PATH="$ruby_path:$PATH" + fi + # Create an alias to easily load the PATH + ynh_ruby_load_path="PATH=$PATH" + + # Sets the local application-specific Ruby version + pushd ${install_dir:-$final_path} + $rbenv_install_dir/bin/rbenv local $ruby_version + popd +} + +# Install a specific version of Ruby +# +# ynh_install_ruby will install the version of Ruby provided as argument by using rbenv. +# +# This helper creates a /etc/profile.d/rbenv.sh that configures PATH environment for rbenv +# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) +# +# Don't forget to execute ruby-dependent command in a login environment +# (e.g. sudo --login option) +# When not possible (e.g. in systemd service definition), please use direct path +# to rbenv shims (e.g. $RBENV_ROOT/shims/bundle) +# +# usage: ynh_install_ruby --ruby_version=ruby_version +# | arg: -v, --ruby_version= - Version of ruby to install. +# +# Requires YunoHost version 3.2.2 or higher. +ynh_install_ruby () { + # Declare an array to define the options of this helper. + local legacy_args=v + local -A args_array=( [v]=ruby_version= ) + local ruby_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Load rbenv path in PATH + local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Ruby prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Move an existing Ruby binary, to avoid to block rbenv + test -x /usr/bin/ruby && mv /usr/bin/ruby /usr/bin/ruby_rbenv + + # Install or update rbenv + mkdir -p $rbenv_install_dir + rbenv="$(command -v rbenv $rbenv_install_dir/bin/rbenv | grep "$rbenv_install_dir/bin/rbenv" | head -1)" + if [ -n "$rbenv" ]; then + pushd "${rbenv%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/rbenv/rbenv.git"; then + ynh_print_info --message="Updating rbenv..." + git pull -q --tags origin master + ynh_ruby_try_bash_extension + else + ynh_print_info --message="Reinstalling rbenv..." + cd .. + ynh_secure_remove --file=$rbenv_install_dir + mkdir -p $rbenv_install_dir + cd $rbenv_install_dir + git init -q + git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 + git checkout -q -b master origin/master + ynh_ruby_try_bash_extension + rbenv=$rbenv_install_dir/bin/rbenv + fi + popd + else + ynh_print_info --message="Installing rbenv..." + pushd $rbenv_install_dir + git init -q + git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 + git checkout -q -b master origin/master + ynh_ruby_try_bash_extension + rbenv=$rbenv_install_dir/bin/rbenv + popd + fi + + mkdir -p "${rbenv_install_dir}/plugins" + + ruby_build="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-install rbenv-install | head -1)" + if [ -n "$ruby_build" ]; then + pushd "${ruby_build%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/rbenv/ruby-build.git"; then + ynh_print_info --message="Updating ruby-build..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing ruby-build..." + git clone -q https://github.com/rbenv/ruby-build.git "${rbenv_install_dir}/plugins/ruby-build" + fi + + rbenv_alias="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)" + if [ -n "$rbenv_alias" ]; then + pushd "${rbenv_alias%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/tpope/rbenv-aliases.git"; then + ynh_print_info --message="Updating rbenv-aliases..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing rbenv-aliases..." + git clone -q https://github.com/tpope/rbenv-aliases.git "${rbenv_install_dir}/plugins/rbenv-aliase" + fi + + rbenv_latest="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)" + if [ -n "$rbenv_latest" ]; then + pushd "${rbenv_latest%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then + ynh_print_info --message="Updating xxenv-latest..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing xxenv-latest..." + git clone -q https://github.com/momo-lab/xxenv-latest.git "${rbenv_install_dir}/plugins/xxenv-latest" + fi + + # Enable caching + mkdir -p "${rbenv_install_dir}/cache" + + # Create shims directory if needed + mkdir -p "${rbenv_install_dir}/shims" + + # Restore /usr/local/bin in PATH + PATH=$CLEAR_PATH + + # And replace the old Ruby binary + test -x /usr/bin/ruby_rbenv && mv /usr/bin/ruby_rbenv /usr/bin/ruby + + # Install the requested version of Ruby + local final_ruby_version=$(rbenv latest --print $ruby_version) + if ! [ -n "$final_ruby_version" ]; then + final_ruby_version=$ruby_version + fi + ynh_print_info --message="Installing Ruby $final_ruby_version" + RUBY_CONFIGURE_OPTS="--disable-install-doc --with-jemalloc" MAKE_OPTS="-j2" rbenv install --skip-existing $final_ruby_version > /dev/null 2>&1 + + # Store ruby_version into the config of this app + ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=ruby_version --value=$final_ruby_version + + # Remove app virtualenv + if rbenv alias --list | grep --quiet "$YNH_APP_INSTANCE_NAME " + then + rbenv alias $YNH_APP_INSTANCE_NAME --remove + fi + + # Create app virtualenv + rbenv alias $YNH_APP_INSTANCE_NAME $final_ruby_version + + # Cleanup Ruby versions + ynh_cleanup_ruby + + # Set environment for Ruby users + echo "#rbenv +export RBENV_ROOT=$rbenv_install_dir +export PATH=\"$rbenv_install_dir/bin:$PATH\" +eval \"\$(rbenv init -)\" +#rbenv" > /etc/profile.d/rbenv.sh + + # Load the environment + eval "$(rbenv init -)" +} + +# Remove the version of Ruby used by the app. +# +# This helper will also cleanup Ruby versions +# +# usage: ynh_remove_ruby +ynh_remove_ruby () { + local ruby_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=ruby_version) + + # Load rbenv path in PATH + local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Ruby prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + rbenv alias $YNH_APP_INSTANCE_NAME --remove + + # Remove the line for this app + ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=ruby_version + + # Cleanup Ruby versions + ynh_cleanup_ruby +} + +# Remove no more needed versions of Ruby used by the app. +# +# This helper will check what Ruby version are no more required, +# and uninstall them +# If no app uses Ruby, rbenv will be also removed. +# +# usage: ynh_cleanup_ruby +ynh_cleanup_ruby () { + + # List required Ruby versions + local installed_apps=$(yunohost app list | grep -oP 'id: \K.*$') + local required_ruby_versions="" + for installed_app in $installed_apps + do + local installed_app_ruby_version=$(ynh_app_setting_get --app=$installed_app --key="ruby_version") + if [[ -n "$installed_app_ruby_version" ]] + then + required_ruby_versions="${installed_app_ruby_version}\n${required_ruby_versions}" + fi + done + + # Remove no more needed Ruby versions + local installed_ruby_versions=$(rbenv versions --bare --skip-aliases | grep -Ev '/') + for installed_ruby_version in $installed_ruby_versions + do + if ! echo ${required_ruby_versions} | grep -q "${installed_ruby_version}" + then + ynh_print_info --message="Removing Ruby-$installed_ruby_version" + $rbenv_install_dir/bin/rbenv uninstall --force $installed_ruby_version + fi + done + + # If none Ruby version is required + if [[ -z "$required_ruby_versions" ]] + then + # Remove rbenv environment configuration + ynh_print_info --message="Removing rbenv" + ynh_secure_remove --file="$rbenv_install_dir" + ynh_secure_remove --file="/etc/profile.d/rbenv.sh" + fi +} + +ynh_ruby_try_bash_extension() { + if [ -x src/configure ]; then + src/configure && make -C src || { + ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." + } + fi +} diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting new file mode 100644 index 000000000..82a5d274e --- /dev/null +++ b/helpers/helpers.v2.1.d/setting @@ -0,0 +1,160 @@ +#!/bin/bash + +# Get an application setting +# +# usage: ynh_app_setting_get --app=app --key=key +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting to get +# +# Requires YunoHost version 2.2.4 or higher. +ynh_app_setting_get() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=ak + local -A args_array=([a]=app= [k]=key=) + local app + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [[ $key =~ (unprotected|protected|skipped)_ ]]; then + yunohost app setting $app $key + else + ynh_app_setting "get" "$app" "$key" + fi +} + +# Set an application setting +# +# usage: ynh_app_setting_set --app=app --key=key --value=value +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting name to set +# | arg: -v, --value= - the setting value to set +# +# Requires YunoHost version 2.2.4 or higher. +ynh_app_setting_set() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=akv + local -A args_array=([a]=app= [k]=key= [v]=value=) + local app + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [[ $key =~ (unprotected|protected|skipped)_ ]]; then + yunohost app setting $app $key -v $value + else + ynh_app_setting "set" "$app" "$key" "$value" + fi +} + +# Delete an application setting +# +# usage: ynh_app_setting_delete --app=app --key=key +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting to delete +# +# Requires YunoHost version 2.2.4 or higher. +ynh_app_setting_delete() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=ak + local -A args_array=([a]=app= [k]=key=) + local app + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then + yunohost app setting $app $key -d + else + ynh_app_setting "delete" "$app" "$key" + fi +} + +# Small "hard-coded" interface to avoid calling "yunohost app" directly each +# time dealing with a setting is needed (which may be so slow on ARM boards) +# +# [internal] +# +ynh_app_setting() { + set +o xtrace # set +x + ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - </dev/null \ + | tr --complement --delete "$filter" \ + | sed --quiet 's/\(.\{'"$length"'\}\).*/\1/p' +} + +# Substitute/replace a string (or expression) by another in a file +# +# usage: ynh_replace_string --match_string=match_string --replace_string=replace_string --target_file=target_file +# | arg: -m, --match_string= - String to be searched and replaced in the file +# | arg: -r, --replace_string= - String that will replace matches +# | arg: -f, --target_file= - File in which the string will be replaced. +# +# As this helper is based on sed command, regular expressions and references to +# sub-expressions can be used (see sed manual page for more information) +# +# Requires YunoHost version 2.6.4 or higher. +ynh_replace_string() { + # Declare an array to define the options of this helper. + local legacy_args=mrf + local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) + local match_string + local replace_string + local target_file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + set +o xtrace # set +x + + local delimit=$'\001' + # Escape the delimiter if it's in the string. + match_string=${match_string//${delimit}/"\\${delimit}"} + replace_string=${replace_string//${delimit}/"\\${delimit}"} + + set -o xtrace # set -x + sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file" +} + +# Substitute/replace a special string by another in a file +# +# usage: ynh_replace_special_string --match_string=match_string --replace_string=replace_string --target_file=target_file +# | arg: -m, --match_string= - String to be searched and replaced in the file +# | arg: -r, --replace_string= - String that will replace matches +# | arg: -t, --target_file= - File in which the string will be replaced. +# +# This helper will use ynh_replace_string, but as you can use special +# characters, you can't use some regular expressions and sub-expressions. +# +# Requires YunoHost version 2.7.7 or higher. +ynh_replace_special_string() { + # Declare an array to define the options of this helper. + local legacy_args=mrf + local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) + local match_string + local replace_string + local target_file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Escape any backslash to preserve them as simple backslash. + match_string=${match_string//\\/"\\\\"} + replace_string=${replace_string//\\/"\\\\"} + + # Escape the & character, who has a special function in sed. + match_string=${match_string//&/"\&"} + replace_string=${replace_string//&/"\&"} + + ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file" +} + +# Sanitize a string intended to be the name of a database +# +# [packagingv1] +# +# usage: ynh_sanitize_dbid --db_name=name +# | arg: -n, --db_name= - name to correct/sanitize +# | ret: the corrected name +# +# example: dbname=$(ynh_sanitize_dbid $app) +# +# Underscorify the string (replace - and . by _) +# +# Requires YunoHost version 2.2.4 or higher. +ynh_sanitize_dbid() { + # Declare an array to define the options of this helper. + local legacy_args=n + local -A args_array=([n]=db_name=) + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # We should avoid having - and . in the name of databases. They are replaced by _ + echo ${db_name//[-.]/_} +} + +# Normalize the url path syntax +# +# [internal] +# +# Handle the slash at the beginning of path and its absence at ending +# Return a normalized url path +# +# examples: +# url_path=$(ynh_normalize_url_path $url_path) +# ynh_normalize_url_path example # -> /example +# ynh_normalize_url_path /example # -> /example +# ynh_normalize_url_path /example/ # -> /example +# ynh_normalize_url_path / # -> / +# +# usage: ynh_normalize_url_path --path_url=path_to_normalize +# | arg: -p, --path_url= - URL path to normalize before using it +# +# Requires YunoHost version 2.6.4 or higher. +ynh_normalize_url_path() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=path_url=) + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." + if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / + path_url="/$path_url" # Add / at begin of path variable + fi + if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then # If the last character is a / and that not the only character. + path_url="${path_url:0:${#path_url}-1}" # Delete the last character + fi + echo $path_url +} diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd new file mode 100644 index 000000000..765c575ef --- /dev/null +++ b/helpers/helpers.v2.1.d/systemd @@ -0,0 +1,189 @@ +#!/bin/bash + +# Create a dedicated systemd config +# +# usage: ynh_add_systemd_config [--service=service] [--template=template] +# | arg: -s, --service= - Service name (optionnal, `$app` by default) +# | arg: -t, --template= - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template) +# +# This will use the template `../conf/.service`. +# +# See the documentation of `ynh_add_config` for a description of the template +# format and how placeholders are replaced with actual variables. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_systemd_config() { + # Declare an array to define the options of this helper. + local legacy_args=stv + local -A args_array=([s]=service= [t]=template=) + local service + local template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + service="${service:-$app}" + template="${template:-systemd.service}" + + ynh_add_config --template="$template" --destination="/etc/systemd/system/$service.service" + + systemctl enable $service --quiet + systemctl daemon-reload +} + +# Remove the dedicated systemd config +# +# usage: ynh_remove_systemd_config [--service=service] +# | arg: -s, --service= - Service name (optionnal, $app by default) +# +# Requires YunoHost version 2.7.2 or higher. +ynh_remove_systemd_config() { + # Declare an array to define the options of this helper. + local legacy_args=s + local -A args_array=([s]=service=) + local service + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local service="${service:-$app}" + + local finalsystemdconf="/etc/systemd/system/$service.service" + if [ -e "$finalsystemdconf" ]; then + ynh_systemd_action --service_name=$service --action=stop + systemctl disable $service --quiet + ynh_secure_remove --file="$finalsystemdconf" + systemctl daemon-reload + fi +} + +# Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started +# +# usage: ynh_systemd_action [--service_name=service_name] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] +# | arg: -n, --service_name= - Name of the service to start. Default : `$app` +# | arg: -a, --action= - Action to perform with systemctl. Default: start +# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. +# | arg: -p, --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` +# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. +# | arg: -e, --length= - Length of the error log displayed for debugging : Default : 20 +# +# Requires YunoHost version 3.5.0 or higher. +ynh_systemd_action() { + # Declare an array to define the options of this helper. + local legacy_args=nalpte + local -A args_array=([n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=) + local service_name + local action + local line_match + local length + local log_path + local timeout + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + service_name="${service_name:-$app}" + action=${action:-start} + line_match=${line_match:-} + length=${length:-20} + log_path="${log_path:-/var/log/$service_name/$service_name.log}" + timeout=${timeout:-300} + + # Manage case of service already stopped + if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service_name; then + return 0 + fi + + # Start to read the log + if [[ -n "$line_match" ]]; then + local templog="$(mktemp)" + # Following the starting of the app in its log + if [ "$log_path" == "systemd" ]; then + # Read the systemd journal + journalctl --unit=$service_name --follow --since=-0 --quiet >"$templog" & + # Get the PID of the journalctl command + local pid_tail=$! + else + # Read the specified log file + tail --follow=name --retry --lines=0 "$log_path" >"$templog" 2>&1 & + # Get the PID of the tail command + local pid_tail=$! + fi + fi + + # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. + if [ "$action" == "reload" ]; then + action="reload-or-restart" + fi + + local time_start="$(date --utc --rfc-3339=seconds | cut -d+ -f1) UTC" + + # If the service fails to perform the action + if ! systemctl $action $service_name; then + # Show syslog for this service + ynh_exec_err journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name + # If a log is specified for this service, show also the content of this log + if [ -e "$log_path" ]; then + ynh_exec_err tail --lines=$length "$log_path" + fi + ynh_clean_check_starting + return 1 + fi + + # Start the timeout and try to find line_match + if [[ -n "${line_match:-}" ]]; then + set +x + local i=0 + local starttime=$(date +%s) + for i in $(seq 1 $timeout); do + # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout + if [ "$log_path" == "systemd" ]; then + # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action + if journalctl --unit=$service_name --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then + ynh_print_info --message="The service $service_name has correctly executed the action ${action}." + break + fi + else + if grep --extended-regexp --quiet "$line_match" "$templog"; then + ynh_print_info --message="The service $service_name has correctly executed the action ${action}." + break + fi + fi + if [ $i -eq 30 ]; then + echo "(this may take some time)" >&2 + fi + # Also check the timeout using actual timestamp, because sometimes for some reason, + # journalctl may take a huge time to run, and we end up waiting literally an entire hour + # instead of 5 min ... + if [[ "$(( $(date +%s) - $starttime))" -gt "$timeout" ]] + then + i=$timeout + break + fi + sleep 1 + done + set -x + if [ $i -ge 3 ]; then + echo "" >&2 + fi + if [ $i -eq $timeout ]; then + ynh_print_warn --message="The service $service_name didn't fully executed the action ${action} before the timeout." + ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" + ynh_exec_warn journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name + if [ -e "$log_path" ]; then + ynh_print_warn --message="\-\-\-" + ynh_exec_warn tail --lines=$length "$log_path" + fi + fi + ynh_clean_check_starting + fi +} + +# Clean temporary process and file used by ynh_check_starting +# +# [internal] +# +# Requires YunoHost version 3.5.0 or higher. +ynh_clean_check_starting() { + if [ -n "${pid_tail:-}" ]; then + # Stop the execution of tail. + kill -SIGTERM $pid_tail 2>&1 + fi + if [ -n "${templog:-}" ]; then + ynh_secure_remove --file="$templog" 2>&1 + fi +} diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user new file mode 100644 index 000000000..e608a3308 --- /dev/null +++ b/helpers/helpers.v2.1.d/user @@ -0,0 +1,196 @@ +#!/bin/bash + +# Check if a YunoHost user exists +# +# usage: ynh_user_exists --username=username +# | arg: -u, --username= - the username to check +# | ret: 0 if the user exists, 1 otherwise. +# +# example: ynh_user_exists 'toto' || echo "User does not exist" +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null +} + +# Retrieve a YunoHost user information +# +# usage: ynh_user_get_info --username=username --key=key +# | arg: -u, --username= - the username to retrieve info from +# | arg: -k, --key= - the key to retrieve +# | ret: the value associate to that key +# +# example: mail=$(ynh_user_get_info --username="toto" --key=mail) +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_get_info() { + # Declare an array to define the options of this helper. + local legacy_args=uk + local -A args_array=([u]=username= [k]=key=) + local username + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user info "$username" --output-as json --quiet | jq -r ".$key" +} + +# Get the list of YunoHost users +# +# usage: ynh_user_list +# | ret: one username per line as strings +# +# example: for u in $(ynh_user_list); do ... ; done +# +# Requires YunoHost version 2.4.0 or higher. +ynh_user_list() { + yunohost user list --output-as json --quiet | jq -r ".users | keys[]" +} + +# Check if a user exists on the system +# +# [packagingv1] +# +# usage: ynh_system_user_exists --username=username +# | arg: -u, --username= - the username to check +# | ret: 0 if the user exists, 1 otherwise. +# +# Requires YunoHost version 2.2.4 or higher. +ynh_system_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent passwd "$username" &>/dev/null +} + +# Check if a group exists on the system +# +# [packagingv1] +# +# usage: ynh_system_group_exists --group=group +# | arg: -g, --group= - the group to check +# | ret: 0 if the group exists, 1 otherwise. +# +# Requires YunoHost version 3.5.0.2 or higher. +ynh_system_group_exists() { + # Declare an array to define the options of this helper. + local legacy_args=g + local -A args_array=([g]=group=) + local group + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent group "$group" &>/dev/null +} + +# Create a system user +# +# [packagingv1] +# +# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell] [--groups="group1 group2"] +# | arg: -u, --username= - Name of the system user that will be create +# | arg: -h, --home_dir= - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home +# | arg: -s, --use_shell - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell +# | arg: -g, --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp) +# +# Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability) : +# ``` +# ynh_system_user_create --username=nextcloud +# ``` +# Create a discourse user using /var/www/discourse as home directory and the default login shell : +# ``` +# ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell +# ``` +# +# Requires YunoHost version 2.6.4 or higher. +ynh_system_user_create() { + # Declare an array to define the options of this helper. + local legacy_args=uhs + local -A args_array=([u]=username= [h]=home_dir= [s]=use_shell [g]=groups=) + local username + local home_dir + local use_shell + local groups + + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + use_shell="${use_shell:-0}" + home_dir="${home_dir:-}" + groups="${groups:-}" + + if ! ynh_system_user_exists "$username"; then # Check if the user exists on the system + # If the user doesn't exist + if [ -n "$home_dir" ]; then # If a home dir is mentioned + local user_home_dir="--home-dir $home_dir" + else + local user_home_dir="--no-create-home" + fi + if [ $use_shell -eq 1 ]; then # If we want a shell for the user + local shell="" # Use default shell + else + local shell="--shell /usr/sbin/nologin" + fi + useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account" + fi + + local group + for group in $groups; do + usermod -a -G "$group" "$username" + done +} + +# Delete a system user +# +# [packagingv1] +# +# usage: ynh_system_user_delete --username=user_name +# | arg: -u, --username= - Name of the system user that will be create +# +# Requires YunoHost version 2.6.4 or higher. +ynh_system_user_delete() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Check if the user exists on the system + if ynh_system_user_exists "$username"; then + deluser $username + else + ynh_print_warn --message="The user $username was not found" + fi + + # Check if the group exists on the system + if ynh_system_group_exists "$username"; then + delgroup $username + fi +} + +# Execute a command as another user +# +# usage: ynh_exec_as $USER COMMAND [ARG ...] +# +# Requires YunoHost version 4.1.7 or higher. +ynh_exec_as() { + local user=$1 + shift 1 + + if [[ $user = $(whoami) ]]; then + eval "$@" + else + sudo -u "$user" "$@" + fi +} diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils new file mode 100644 index 000000000..631e154e2 --- /dev/null +++ b/helpers/helpers.v2.1.d/utils @@ -0,0 +1,1104 @@ +#!/bin/bash + +YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)} + +# Handle script crashes / failures +# +# [internal] +# +# usage: +# ynh_exit_properly is used only by the helper ynh_abort_if_errors. +# You should not use it directly. +# Instead, add to your script: +# ynh_clean_setup () { +# instructions... +# } +# +# This function provide a way to clean some residual of installation that not managed by remove script. +# +# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script +# +# Requires YunoHost version 2.6.4 or higher. +ynh_exit_properly() { + local exit_code=$? + + if [[ "${YNH_APP_ACTION:-}" =~ ^install$|^upgrade$|^restore$ ]] + then + rm -rf "/var/cache/yunohost/download/" + fi + + if [ "$exit_code" -eq 0 ]; then + exit 0 # Exit without error if the script ended correctly + fi + + trap '' EXIT # Ignore new exit signals + # Do not exit anymore if a command fail or if a variable is empty + set +o errexit # set +e + set +o nounset # set +u + + # Small tempo to avoid the next message being mixed up with other DEBUG messages + sleep 0.5 + + if type -t ynh_clean_setup >/dev/null; then # Check if the function exist in the app script. + ynh_clean_setup # Call the function to do specific cleaning for the app. + fi + + # Exit with error status + # We don't call ynh_die basically to avoid unecessary 10-ish + # debug lines about parsing args and stuff just to exit 1.. + exit 1 +} + +# Exits if an error occurs during the execution of the script. +# +# [packagingv1] +# +# usage: ynh_abort_if_errors +# +# This configure the rest of the script execution such that, if an error occurs +# or if an empty variable is used, the execution of the script stops immediately +# and a call to `ynh_clean_setup` is triggered if it has been defined by your script. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_abort_if_errors() { + set -o errexit # set -e; Exit if a command fail + set -o nounset # set -u; And if a variable is used unset + trap ynh_exit_properly EXIT # Capturing exit signals on shell script +} + +# When running an app script with packaging format >= 2, auto-enable ynh_abort_if_errors except for remove script +if [[ "${YNH_CONTEXT:-}" != "regenconf" ]] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 && [[ ${YNH_APP_ACTION} != "remove" ]] +then + ynh_abort_if_errors +fi + +# Download, check integrity, uncompress and patch the source from app.src +# +# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] +# | arg: -d, --dest_dir= - Directory where to setup sources +# | arg: -s, --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise +# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders) +# | arg: -r, --full_replace= - Remove previous sources before installing new sources (can be 1 or 0, default to 0) +# +# #### New 'sources' resources +# +# (See also the resources documentation which may be more complete?) +# +# This helper will read infos from the 'sources' resources in the manifest.toml of the app +# and expect a structure like: +# +# ```toml +# [resources.sources] +# [resources.sources.main] +# url = "https://some.address.to/download/the/app/archive" +# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL +# ``` +# +# ##### Optional flags +# +# ```text +# format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract +# "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract +# "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract +# "whatever" # an arbitrary value, not really meaningful except to imply that the file won't be extracted +# +# in_subdir = true # default, there's an intermediate subdir in the archive before accessing the actual files +# false # sources are directly in the archive root +# n # (special cases) an integer representing a number of subdirs levels to get rid of +# +# extract = true # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ... +# = false # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset. +# # in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value +# +# rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical +# platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for +# ``` +# +# You may also define assets url and checksum per-architectures such as: +# ```toml +# [resources.sources] +# [resources.sources.main] +# amd64.url = "https://some.address.to/download/the/app/archive/when/amd64" +# amd64.sha256 = "0123456789abcdef" +# armhf.url = "https://some.address.to/download/the/app/archive/when/armhf" +# armhf.sha256 = "fedcba9876543210" +# ``` +# +# In which case ynh_setup_source --dest_dir="$install_dir" will automatically pick the appropriate source depending on the arch +# +# +# +# #### Legacy format '.src' +# +# This helper will read `conf/${source_id}.src`, download and install the sources. +# +# The src file need to contains: +# ``` +# SOURCE_URL=Address to download the app archive +# SOURCE_SUM=Sha256 sum +# SOURCE_FORMAT=tar.gz +# SOURCE_IN_SUBDIR=false +# SOURCE_FILENAME=example.tar.gz +# SOURCE_EXTRACT=(true|false) +# SOURCE_PLATFORM=linux/arm64/v8 +# ``` +# +# The helper will: +# - Download the specific URL if there is no local archive +# - Check the integrity with the specific sha256 sum +# - Uncompress the archive to `$dest_dir`. +# - If `in_subdir` is true, the first level directory of the archive will be removed. +# - If `in_subdir` is a numeric value, the N first level directories will be removed. +# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` +# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir +# +# Requires YunoHost version 2.6.4 or higher. +ynh_setup_source() { + # Declare an array to define the options of this helper. + local legacy_args=dsk + local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) + local dest_dir + local source_id + local keep + local full_replace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + keep="${keep:-}" + full_replace="${full_replace:-0}" + + if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null + then + source_id="${source_id:-main}" + local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") + if jq -re ".url" <<< "$sources_json" + then + local arch_prefix="" + else + local arch_prefix=".$YNH_ARCH" + fi + + local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" + local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" + local src_sumprg="sha256sum" + local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" + local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" + local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" + local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" + local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" + + [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" + + if [[ -z "$src_format" ]] + then + if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] + then + src_format="zip" + elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] + then + src_format="tar.gz" + elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] + then + src_format="tar.xz" + elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] + then + src_format="tar.bz2" + elif [[ -z "$src_extract" ]] + then + src_extract="false" + fi + fi + else + source_id="${source_id:-app}" + local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" + + # Load value from configuration file (see above for a small doc about this file + # format) + local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_rename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_platform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + fi + + # Default value + src_sumprg=${src_sumprg:-sha256sum} + src_in_subdir=${src_in_subdir:-true} + src_format=${src_format:-tar.gz} + src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') + src_extract=${src_extract:-true} + + if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] + then + ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" + fi + + + # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... + local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" + + # Gotta use this trick with 'dirname' because source_id may contain slashes x_x + mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) + src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" + + if [ "$src_format" = "docker" ]; then + src_platform="${src_platform:-"linux/$YNH_ARCH"}" + else + if test -e "$local_src"; then + cp $local_src $src_filename + fi + + [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" + + # If the file was prefetched but somehow doesn't match the sum, rm and redownload it + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + rm -f "$src_filename" + fi + + # Only redownload the file if it wasnt prefetched + if [ ! -e "$src_filename" ] + then + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ + || ynh_die --message="$out" + fi + + # Check the control sum + if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_size="$(du -hs ${src_filename} | cut --fields=1)" + rm -f ${src_filename} + ynh_die --message="Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." + fi + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ + if [ -n "$keep" ] && [ -e "$dest_dir" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + mkdir -p $keep_dir + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$dest_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" + cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" + fi + done + fi + + if [ "$full_replace" -eq 1 ]; then + ynh_secure_remove --file="$dest_dir" + fi + + # Extract source into the app dir + mkdir --parents "$dest_dir" + + if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then + _ynh_apply_default_permissions $dest_dir + fi + if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then + _ynh_apply_default_permissions $dest_dir + fi + + if [[ "$src_extract" == "false" ]]; then + if [[ -z "$src_rename" ]] + then + mv $src_filename $dest_dir + else + mv $src_filename $dest_dir/$src_rename + fi + elif [[ "$src_format" == "docker" ]]; then + "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1 + elif [[ "$src_format" == "zip" ]]; then + # Zip format + # Using of a temp directory, because unzip doesn't manage --strip-components + if $src_in_subdir; then + local tmp_dir=$(mktemp --directory) + unzip -quo $src_filename -d "$tmp_dir" + cp --archive $tmp_dir/*/. "$dest_dir" + ynh_secure_remove --file="$tmp_dir" + else + unzip -quo $src_filename -d "$dest_dir" + fi + ynh_secure_remove --file="$src_filename" + else + local strip="" + if [ "$src_in_subdir" != "false" ]; then + if [ "$src_in_subdir" == "true" ]; then + local sub_dirs=1 + else + local sub_dirs="$src_in_subdir" + fi + strip="--strip-components $sub_dirs" + fi + if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then + tar --extract --file=$src_filename --directory="$dest_dir" $strip + else + ynh_die --message="Archive format unrecognized." + fi + ynh_secure_remove --file="$src_filename" + fi + + # Apply patches + if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then + local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) + if (($(find $patches_folder -type f -name "${source_id}-*.patch" 2>/dev/null | wc --lines) > "0")); then + pushd "$dest_dir" + for p in $patches_folder/${source_id}-*.patch; do + echo $p + patch --strip=1 <$p || ynh_print_warn --message="Packagers /!\\ patch $p failed to apply" + done + popd + fi + fi + + # Add supplementary files + if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then + cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir" + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + if [ -n "$keep" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$keep_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" + + # We add "--no-target-directory" (short option is -T) to handle the special case + # when we "keep" a folder, but then the new setup already contains the same dir (but possibly empty) + # in which case a regular "cp" will create a copy of the directory inside the directory ... + # resulting in something like /var/www/$app/data/data instead of /var/www/$app/data + # cf https://unix.stackexchange.com/q/94831 for a more elaborate explanation on the option + cp --archive --no-target-directory "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" + fi + done + fi + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ +} + +# Curl abstraction to help with POST requests to local pages (such as installation forms) +# +# usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ... +# | arg: page_uri - Path (relative to `$path_url`) of the page where POST data will be sent +# | arg: key1=value1 - (Optionnal) POST key and corresponding value +# | arg: key2=value2 - (Optionnal) Another POST key and corresponding value +# | arg: ... - (Optionnal) More POST keys and values +# +# example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2" +# +# For multiple calls, cookies are persisted between each call for the same app +# +# `$domain` and `$path_url` should be defined externally (and correspond to the domain.tld and the /path (of the app?)) +# +# Requires YunoHost version 2.6.4 or higher. +ynh_local_curl() { + # Define url of page to curl + local local_page=$(ynh_normalize_url_path $1) + local full_path=$path_url$local_page + + if [ "${path_url}" == "/" ]; then + full_path=$local_page + fi + + local full_page_url=https://localhost$full_path + + # Concatenate all other arguments with '&' to prepare POST data + local POST_data="" + local arg="" + for arg in "${@:2}"; do + POST_data="${POST_data}${arg}&" + done + if [ -n "$POST_data" ]; then + # Add --data arg and remove the last character, which is an unecessary '&' + POST_data="--data ${POST_data::-1}" + fi + + # Wait untils nginx has fully reloaded (avoid curl fail with http2) + sleep 2 + + local cookiefile=/tmp/ynh-$app-cookie.txt + touch $cookiefile + chown root $cookiefile + chmod 700 $cookiefile + + # Temporarily enable visitors if needed... + local visitors_enabled=$(ynh_permission_has_user "main" "visitors" && echo yes || echo no) + if [[ $visitors_enabled == "no" ]]; then + ynh_permission_update --permission "main" --add "visitors" + fi + + # Curl the URL + curl --silent --show-error --insecure --location --header "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" --cookie-jar $cookiefile --cookie $cookiefile + + if [[ $visitors_enabled == "no" ]]; then + ynh_permission_update --permission "main" --remove "visitors" + fi +} + +# Create a dedicated config file from a template +# +# usage: ynh_add_config --template="template" --destination="destination" +# | arg: -t, --template= - Template config file to use +# | arg: -d, --destination= - Destination of the config file +# +# examples: +# ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" +# ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" +# +# The template can be by default the name of a file in the conf directory +# of a YunoHost Package, a relative path or an absolute path. +# +# The helper will use the template `template` to generate a config file +# `destination` by replacing the following keywords with global variables +# that should be defined before calling this helper : +# ``` +# __PATH__ by $path_url +# __NAME__ by $app +# __NAMETOCHANGE__ by $app +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# ``` +# And any dynamic variables that should be defined before calling this helper like: +# ``` +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# ``` +# +# The helper will verify the checksum and backup the destination file +# if it's different before applying the new template. +# +# And it will calculate and store the destination file checksum +# into the app settings when configuration is done. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_config() { + # Declare an array to define the options of this helper. + local legacy_args=tdv + local -A args_array=([t]=template= [d]=destination=) + local template + local destination + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local template_path + + if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then + template_path="$YNH_APP_BASEDIR/conf/$template" + elif [ -f "$template" ]; then + template_path=$template + else + ynh_die --message="The provided template $template doesn't exist" + fi + + ynh_backup_if_checksum_is_different --file="$destination" + + # Make sure to set the permissions before we copy the file + # This is to cover a case where an attacker could have + # created a file beforehand to have control over it + # (cp won't overwrite ownership / modes by default...) + touch $destination + chown root:root $destination + chmod 640 $destination + + cp -f "$template_path" "$destination" + + _ynh_apply_default_permissions $destination + + ynh_replace_vars --file="$destination" + + ynh_store_file_checksum --file="$destination" +} + +# Replace variables in a file +# +# [internal] +# +# usage: ynh_replace_vars --file="file" +# | arg: -f, --file= - File where to replace variables +# +# The helper will replace the following keywords with global variables +# that should be defined before calling this helper : +# __PATH__ by $path_url +# __NAME__ by $app +# __NAMETOCHANGE__ by $app +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# +# And any dynamic variables that should be defined before calling this helper like: +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# +# Requires YunoHost version 4.1.0 or higher. +ynh_replace_vars() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Replace specific YunoHost variables + if test -n "${path_url:-}"; then + # path_url_slash_less is path_url, or a blank value if path_url is only '/' + local path_url_slash_less=${path_url%/} + ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file" + ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" + fi + if test -n "${app:-}"; then + ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$file" + ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file" + ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" + fi + # Legacy + if test -n "${final_path:-}"; then + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file" + ynh_replace_string --match_string="__INSTALL_DIR__" --replace_string="$final_path" --target_file="$file" + fi + # Legacy / Packaging v1 only + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2 && test -n "${YNH_PHP_VERSION:-}"; then + ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file" + fi + if test -n "${ynh_node_load_PATH:-}"; then + ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" + fi + + # Replace others variables + + # List other unique (__ __) variables in $file + local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g")) + + set +o xtrace # set +x + + # Do the replacement + local delimit=@ + for one_var in "${uniques_vars[@]}"; do + # Validate that one_var is indeed defined + # -v checks if the variable is defined, for example: + # -v FOO tests if $FOO is defined + # -v $FOO tests if ${!FOO} is defined + # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 + [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" + + # Escape delimiter in match/replace string + match_string="__${one_var^^}__" + match_string=${match_string//${delimit}/"\\${delimit}"} + replace_string="${!one_var}" + replace_string=${replace_string//\\/\\\\} + replace_string=${replace_string//${delimit}/"\\${delimit}"} + + # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs) + sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file" + done + set -o xtrace # set -x +} + +# Get a value from heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_read_var_in_file --file=PATH --key=KEY +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to get +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# This helpers match several var affectation use case in several languages +# We don't use jq or equivalent to keep comments and blank space in files +# This helpers work line by line, it is not able to work correctly +# if you have several identical keys in your files +# +# Example of line this helpers can managed correctly +# .yml +# title: YunoHost documentation +# email: 'yunohost@yunohost.org' +# .json +# "theme": "colib'ris", +# "port": 8102 +# "some_boolean": false, +# "user": null +# .ini +# some_boolean = On +# action = "Clear" +# port = 20 +# .php +# $user= +# user => 20 +# .py +# USER = 8102 +# user = 'https://donate.local' +# CUSTOM['user'] = 'YunoHost' +# +# Requires YunoHost version 4.3 or higher. +ynh_read_var_in_file() { + # Declare an array to define the options of this helper. + local legacy_args=fka + local -A args_array=([f]=file= [k]=key= [a]=after=) + local file + local key + local after + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + after="${after:-}" + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local line_number=1 + if [[ -n "$after" ]]; then + line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + echo YNH_NULL + return 0 + fi + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + + local first_char="${expression:0:1}" + if [[ "$first_char" == '"' ]]; then + echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + elif [[ "$first_char" == "'" ]]; then + echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + else + echo "$expression" + fi + set -o xtrace # set -x +} + +# Set a value into heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to set +# | arg: -v, --value= - the value to set +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# Requires YunoHost version 4.3 or higher. +ynh_write_var_in_file() { + # Declare an array to define the options of this helper. + local legacy_args=fkva + local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) + local file + local key + local value + local after + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + after="${after:-}" + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local after_line_number=1 + if [[ -n "$after" ]]; then + after_line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$after_line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$after_line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + return 1 + fi + local value_line_number="$(tail +$after_line_number ${file} | grep -m1 -n -i -P $var_part'\K.*$' | cut -d: -f1)" + value_line_number=$((after_line_number + value_line_number)) + local range="${after_line_number},${value_line_number} " + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + endline=${expression_with_comment#"$expression"} + endline="$(echo "$endline" | sed 's/\\/\\\\/g')" + value="$(echo "$value" | sed 's/\\/\\\\/g')" + value=${value//&/"\&"} + local first_char="${expression:0:1}" + delimiter=$'\001' + if [[ "$first_char" == '"' ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # So we need \\\\ to go through 2 sed + value="$(echo "$value" | sed 's/"/\\\\"/g')" + sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file} + elif [[ "$first_char" == "'" ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # However double quotes implies to double \\ to + # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str + value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" + sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file} + else + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]]; then + value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' + fi + if [[ "$ext" =~ ^yaml|yml$ ]]; then + value=" $value" + fi + sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file} + fi + set -o xtrace # set -x +} + +# Render templates with Jinja2 +# +# [internal] +# +# Attention : Variables should be exported before calling this helper to be +# accessible inside templates. +# +# usage: ynh_render_template some_template output_path +# | arg: some_template - Template file to be rendered +# | arg: output_path - The path where the output will be redirected to +ynh_render_template() { + local template_path=$1 + local output_path=$2 + mkdir -p "$(dirname $output_path)" + # Taken from https://stackoverflow.com/a/35009576 + python3 -c 'import os, sys, jinja2; sys.stdout.write( + jinja2.Template(sys.stdin.read() + ).render(os.environ));' <$template_path >$output_path +} + +# Fetch the Debian release codename +# +# [packagingv1] +# +# usage: ynh_get_debian_release +# | ret: The Debian release codename (i.e. jessie, stretch, ...) +# +# Requires YunoHost version 2.7.12 or higher. +ynh_get_debian_release() { + echo $(lsb_release --codename --short) +} + +_acceptable_path_to_delete() { + local file=$1 + + local forbidden_paths=$(ls -d / /* /{var,home,usr}/* /etc/{default,sudoers.d,yunohost,cron*} /etc/yunohost/{apps,domains,hooks.d} /opt/yunohost 2> /dev/null) + + # Legacy : A couple apps still have data in /home/$app ... + if [[ -n "${app:-}" ]] + then + forbidden_paths=$(echo "$forbidden_paths" | grep -v "/home/$app") + fi + + # Use realpath to normalize the path .. + # i.e convert ///foo//bar//..///baz//// to /foo/baz + file=$(realpath --no-symlinks "$file") + if [ -z "$file" ] || grep -q -x -F "$file" <<< "$forbidden_paths"; then + return 1 + else + return 0 + fi +} + +# Remove a file or a directory securely +# +# usage: ynh_secure_remove --file=path_to_remove +# | arg: -f, --file= - File or directory to remove +# +# Requires YunoHost version 2.6.4 or higher. +ynh_secure_remove() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + set +o xtrace # set +x + + if [ $# -ge 2 ]; then + ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." + fi + + if [[ -z "$file" ]]; then + ynh_print_warn --message="ynh_secure_remove called with empty argument, ignoring." + elif [[ ! -e $file ]]; then + ynh_print_info --message="'$file' wasn't deleted because it doesn't exist." + elif ! _acceptable_path_to_delete "$file"; then + ynh_print_warn --message="Not deleting '$file' because it is not an acceptable path to delete." + else + rm --recursive "$file" + fi + + set -o xtrace # set -x +} + +# Read the value of a key in a ynh manifest file +# +# usage: ynh_read_manifest --manifest="manifest.json" --key="key" +# | arg: -m, --manifest= - Path of the manifest to read +# | arg: -k, --key= - Name of the key to find +# | ret: the value associate to that key +# +# Requires YunoHost version 3.5.0 or higher. +ynh_read_manifest() { + # Declare an array to define the options of this helper. + local legacy_args=mk + local -A args_array=([m]=manifest= [k]=manifest_key=) + local manifest + local manifest_key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [ ! -e "${manifest:-}" ]; then + # If the manifest isn't found, try the common place for backup and restore script. + if [ -e "$YNH_APP_BASEDIR/manifest.json" ] + then + manifest="$YNH_APP_BASEDIR/manifest.json" + elif [ -e "$YNH_APP_BASEDIR/manifest.toml" ] + then + manifest="$YNH_APP_BASEDIR/manifest.toml" + else + ynh_die --message "No manifest found !?" + fi + fi + + if echo "$manifest" | grep -q '\.json$' + then + jq ".$manifest_key" "$manifest" --raw-output + else + cat "$manifest" | python3 -c 'import json, toml, sys; print(json.dumps(toml.load(sys.stdin)))' | jq ".$manifest_key" --raw-output + fi +} + +# Read the upstream version from the manifest or `$YNH_APP_MANIFEST_VERSION` +# +# usage: ynh_app_upstream_version [--manifest="manifest.json"] +# | arg: -m, --manifest= - Path of the manifest to read +# | ret: the version number of the upstream app +# +# If the `manifest` is not specified, the envvar `$YNH_APP_MANIFEST_VERSION` will be used. +# +# The version number in the manifest is defined by `~ynh`. +# +# For example, if the manifest contains `4.3-2~ynh3` the function will return `4.3-2` +# +# Requires YunoHost version 3.5.0 or higher. +ynh_app_upstream_version() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=manifest=) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + manifest="${manifest:-}" + + if [[ "$manifest" != "" ]] && [[ -e "$manifest" ]]; then + version_key_=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + else + version_key_=$YNH_APP_MANIFEST_VERSION + fi + + echo "${version_key_/~ynh*/}" +} + +# Read package version from the manifest +# +# [internal] +# +# usage: ynh_app_package_version [--manifest="manifest.json"] +# | arg: -m, --manifest= - Path of the manifest to read +# | ret: the version number of the package +# +# The version number in the manifest is defined by `~ynh`. +# +# For example, if the manifest contains `4.3-2~ynh3` the function will return `3` +# +# Requires YunoHost version 3.5.0 or higher. +ynh_app_package_version() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=manifest=) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + version_key_=$YNH_APP_MANIFEST_VERSION + echo "${version_key_/*~ynh/}" +} + +# Checks the app version to upgrade with the existing app version and returns: +# +# usage: ynh_check_app_version_changed +# | ret: `UPGRADE_APP` if the upstream version changed, `UPGRADE_PACKAGE` otherwise. +# +# This helper should be used to avoid an upgrade of an app, or the upstream part +# of it, when it's not needed +# +# Requires YunoHost version 3.5.0 or higher. +ynh_check_app_version_changed() { + local return_value=${YNH_APP_UPGRADE_TYPE} + + if [ "$return_value" == "UPGRADE_SAME" ] || [ "$return_value" == "DOWNGRADE" ]; then + return_value="UPGRADE_APP" + fi + + echo $return_value +} + +# Compare the current package version against another version given as an argument. +# +# usage: ynh_compare_current_package_version --comparison (lt|le|eq|ne|ge|gt) --version +# | arg: --comparison - Comparison type. Could be : `lt` (lower than), `le` (lower or equal), `eq` (equal), `ne` (not equal), `ge` (greater or equal), `gt` (greater than) +# | arg: --version - The version to compare. Need to be a version in the yunohost package version type (like `2.3.1~ynh4`) +# | ret: 0 if the evaluation is true, 1 if false. +# +# example: ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1 +# +# This helper is usually used when we need to do some actions only for some old package versions. +# +# Generally you might probably use it as follow in the upgrade script : +# ``` +# if ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1 +# then +# # Do something that is needed for the package version older than 2.3.2~ynh1 +# fi +# ``` +# +# Requires YunoHost version 3.8.0 or higher. +ynh_compare_current_package_version() { + local legacy_args=cv + declare -Ar args_array=([c]=comparison= [v]=version=) + local version + local comparison + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local current_version=$YNH_APP_CURRENT_VERSION + + # Check the syntax of the versions + if [[ ! $version =~ '~ynh' ]] || [[ ! $current_version =~ '~ynh' ]]; then + ynh_die --message="Invalid argument for version." + fi + + # Check validity of the comparator + if [[ ! $comparison =~ (lt|le|eq|ne|ge|gt) ]]; then + ynh_die --message="Invalid comparator must be : lt, le, eq, ne, ge, gt" + fi + + # Return the return value of dpkg --compare-versions + dpkg --compare-versions $current_version $comparison $version +} + +# Check if we should enforce sane default permissions (= disable rwx for 'others') +# on file/folders handled with ynh_setup_source and ynh_add_config +# +# [internal] +# +# Having a file others-readable or a folder others-executable(=enterable) +# is a security risk comparable to "chmod 777" +# +# Configuration files may contain secrets. Or even just being able to enter a +# folder may allow an attacker to do nasty stuff (maybe a file or subfolder has +# some write permission enabled for 'other' and the attacker may edit the +# content or create files as leverage for priviledge escalation ...) +# +# The sane default should be to set ownership to $app:$app. +# In specific case, you may want to set the ownership to $app:www-data +# for example if nginx needs access to static files. +# +_ynh_apply_default_permissions() { + local target=$1 + + local ynh_requirement=$(ynh_read_manifest --manifest_key="requirements.yunohost" | tr -d '<>= ') + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then + chmod o-rwx $target + chmod g-w $target + chown -R root:root $target + if ynh_system_user_exists $app; then + chown $app:$app $target + fi + fi + + # Crons should be owned by root + # Also we don't want systemd conf, nginx conf or others stuff to be owned by the app, + # otherwise they could self-edit their own systemd conf and escalate privilege + if echo "$target" | grep -q '^/etc/cron\|/etc/php\|/etc/nginx/conf.d\|/etc/fail2ban\|/etc/systemd/system' + then + chmod 400 $target + chown root:root $target + fi +} + +int_to_bool() { + sed -e 's/^1$/True/g' -e 's/^0$/False/g' +} + +toml_to_json() { + python3 -c 'import toml, json, sys; print(json.dumps(toml.load(sys.stdin)))' +} diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE b/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE new file mode 100644 index 000000000..986360f1a --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2020-2023, Jeremy Lin + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md b/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md new file mode 100644 index 000000000..4c4fa301f --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md @@ -0,0 +1 @@ +This is taken from https://github.com/jjlin/docker-image-extract, under MIT license. \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract b/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract new file mode 100755 index 000000000..b5dfdb7a7 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract @@ -0,0 +1,288 @@ +#!/bin/sh +# +# This script pulls and extracts all files from an image in Docker Hub. +# +# Copyright (c) 2020-2023, Jeremy Lin +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +PLATFORM_DEFAULT="linux/amd64" +PLATFORM="${PLATFORM_DEFAULT}" +OUT_DIR="./output" + +usage() { + echo "This script pulls and extracts all files from an image in Docker Hub." + echo + echo "$0 [OPTIONS...] IMAGE[:REF]" + echo + echo "IMAGE can be a community user image (like 'some-user/some-image') or a" + echo "Docker official image (like 'hello-world', which contains no '/')." + echo + echo "REF is either a tag name or a full SHA-256 image digest (with a 'sha256:' prefix)." + echo "The default ref is the 'latest' tag." + echo + echo "Options:" + echo + echo " -p PLATFORM Pull image for the specified platform (default: ${PLATFORM})" + echo " For a given image on Docker Hub, the 'Tags' tab lists the" + echo " platforms supported for that image." + echo " -o OUT_DIR Extract image to the specified output dir (default: ${OUT_DIR})" + echo " -h Show help with usage examples" +} + +usage_detailed() { + usage + echo + echo "Examples:" + echo + echo "# Pull and extract all files in the 'hello-world' image tagged 'latest'." + echo "\$ $0 hello-world:latest" + echo + echo "# Same as above; ref defaults to the 'latest' tag." + echo "\$ $0 hello-world" + echo + echo "# Pull the 'hello-world' image for the 'linux/arm64/v8' platform." + echo "\$ $0 -p linux/arm64/v8 hello-world" + echo + echo "# Pull an image by digest." + echo "\$ $0 hello-world:sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042" +} + +if [ $# -eq 0 ]; then + usage_detailed + exit 0 +fi + +while getopts ':ho:p:' opt; do + case $opt in + o) + OUT_DIR="${OPTARG}" + ;; + p) + PLATFORM="${OPTARG}" + ;; + h) + usage_detailed + exit 0 + ;; + \?) + echo "ERROR: Invalid option '-$OPTARG'." + echo + usage + exit 1 + ;; + \:) echo "ERROR: Argument required for option '-$OPTARG'." + echo + usage + exit 1 + ;; + esac +done +shift $(($OPTIND - 1)) + +if [ $# -eq 0 ]; then + echo "ERROR: Image to pull must be specified." + echo + usage + exit 1 +fi + +if [ -e "${OUT_DIR}" ]; then + if [ -d "${OUT_DIR}" ]; then + echo "WARNING: Output dir already exists. If it contains a previous extracted image," + echo "there may be errors when trying to overwrite files with read-only permissions." + echo + else + echo "ERROR: Output dir already exists, but is not a directory." + exit 1 + fi +fi + +have_curl() { + command -v curl >/dev/null +} + +have_wget() { + command -v wget >/dev/null +} + +if ! have_curl && ! have_wget; then + echo "This script requires either curl or wget." + exit 1 +fi + +image_spec="$1" +image="${image_spec%%:*}" +if [ "${image#*/}" = "${image}" ]; then + # Docker official images are in the 'library' namespace. + image="library/${image}" +fi +ref="${image_spec#*:}" +if [ "${ref}" = "${image_spec}" ]; then + echo "Defaulting ref to tag 'latest'..." + ref=latest +fi + +# Split platform (OS/arch/variant) into separate variables. +# A platform specifier doesn't always include the `variant` component. +OLD_IFS="${IFS}" +IFS=/ read -r OS ARCH VARIANT <":"" (assumes key/val won't contain double quotes). + # The colon may have whitespace on either side. + grep -o "\"${key}\"[[:space:]]*:[[:space:]]*\"[^\"]\+\"" | + # Extract just by deleting the last '"', and then greedily deleting + # everything up to '"'. + sed -e 's/"$//' -e 's/.*"//' +} + +# Fetch a URL to stdout. Up to two header arguments may be specified: +# +# fetch [name1: value1] [name2: value2] +# +fetch() { + if have_curl; then + if [ $# -eq 2 ]; then + set -- -H "$2" "$1" + elif [ $# -eq 3 ]; then + set -- -H "$2" -H "$3" "$1" + fi + curl -sSL "$@" + else + if [ $# -eq 2 ]; then + set -- --header "$2" "$1" + elif [ $# -eq 3 ]; then + set -- --header "$2" --header "$3" "$1" + fi + wget -qO- "$@" + fi +} + +# https://docs.docker.com/docker-hub/api/latest/#tag/repositories +manifest_list_url="https://hub.docker.com/v2/repositories/${image}/tags/${ref}" + +# If the ref is already a SHA-256 image digest, then we don't need to look up anything. +if [ -z "${ref##sha256:*}" ]; then + digest="${ref}" +else + echo "Getting multi-arch manifest list..." + NL=' +' + digest=$(fetch "${manifest_list_url}" | + # Break up the single-line JSON output into separate lines by adding + # newlines before and after the chars '[', ']', '{', and '}'. + # This uses the \${NL} syntax because some BSD variants of sed don't + # support \n syntax in the replacement string, but instead require + # a literal newline preceded by a backslash. + sed -e 's/\([][{}]\)/\'"${NL}"'\1\'"${NL}"'/g' | + # Extract the "images":[...] list. + sed -n '/"images":/,/]/ p' | + # Each image's details are now on a separate line, e.g. + # "architecture":"arm64","features":"","variant":"v8","digest":"sha256:054c85801c4cb41511b176eb0bf13a2c4bbd41611ddd70594ec3315e88813524","os":"linux","os_features":"","os_version":null,"size":828724,"status":"active","last_pulled":"2022-09-02T22:46:48.240632Z","last_pushed":"2022-09-02T00:42:45.69226Z" + # The image details are interspersed with lines of stray punctuation, + # so grep for an arbitrary string that must be in these lines. + grep architecture | + # Search for an image that matches the platform. + while read -r image; do + # Arch is probably most likely to be unique, so check that first. + arch="$(echo ${image} | extract 'architecture')" + if [ "${arch}" != "${ARCH}" ]; then continue; fi + + os="$(echo ${image} | extract 'os')" + if [ "${os}" != "${OS}" ]; then continue; fi + + variant="$(echo ${image} | extract 'variant')" + if [ "${variant}" = "${VARIANT}" ]; then + echo ${image} | extract 'digest' + break + fi + done) + + if [ -n "${digest}" ]; then + echo "Platform ${PLATFORM} resolved to '${digest}'..." + else + echo "No image digest found. Verify that the image, ref, and platform are valid." + exit 1 + fi +fi + +# https://docs.docker.com/registry/spec/auth/token/#how-to-authenticate +api_token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" + +# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-an-image-manifest +manifest_url="https://registry-1.docker.io/v2/${image}/manifests/${digest}" + +# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-a-layer +blobs_base_url="https://registry-1.docker.io/v2/${image}/blobs" + +echo "Getting API token..." +token=$(fetch "${api_token_url}" | extract 'token') +auth_header="Authorization: Bearer $token" + +# https://github.com/distribution/distribution/blob/main/docs/spec/manifest-v2-2.md +docker_manifest_v2="application/vnd.docker.distribution.manifest.v2+json" + +# https://github.com/opencontainers/image-spec/blob/main/manifest.md +oci_manifest_v1="application/vnd.oci.image.manifest.v1+json" + +# Docker Hub can return either type of manifest format. Most images seem to +# use the Docker format for now, but the OCI format will likely become more +# common as features that require that format become enabled by default +# (e.g., https://github.com/docker/build-push-action/releases/tag/v3.3.0). +accept_header="Accept: ${docker_manifest_v2},${oci_manifest_v1}" + +echo "Getting image manifest for $image:$ref..." +layers=$(fetch "${manifest_url}" "${auth_header}" "${accept_header}" | + # Extract `digest` values only after the `layers` section appears. + sed -n '/"layers":/,$ p' | + extract 'digest') + +if [ -z "${layers}" ]; then + echo "No layers returned. Verify that the image and ref are valid." + exit 1 +fi + +mkdir -p "${OUT_DIR}" + +for layer in $layers; do + hash="${layer#sha256:}" + echo "Fetching and extracting layer ${hash}..." + fetch "${blobs_base_url}/${layer}" "${auth_header}" | gzip -d | tar -C "${OUT_DIR}" -xf - + # Ref: https://github.com/moby/moby/blob/master/image/spec/v1.2.md#creating-an-image-filesystem-changeset + # https://github.com/moby/moby/blob/master/pkg/archive/whiteouts.go + # Search for "whiteout" files to indicate files deleted in this layer. + OLD_IFS="${IFS}" + find "${OUT_DIR}" -name '.wh.*' | while IFS= read -r f; do + dir="${f%/*}" + wh_file="${f##*/}" + file="${wh_file#.wh.}" + # Delete both the whiteout file and the whited-out file. + rm -rf "${dir}/${wh_file}" "${dir}/${file}" + done + IFS="${OLD_IFS}" +done + +echo "Image contents extracted into ${OUT_DIR}." \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/n/LICENSE b/helpers/helpers.v2.1.d/vendor/n/LICENSE new file mode 100644 index 000000000..8e04e8467 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/n/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 TJ Holowaychuk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/helpers/helpers.v2.1.d/vendor/n/README.md b/helpers/helpers.v2.1.d/vendor/n/README.md new file mode 100644 index 000000000..9a29a3936 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/n/README.md @@ -0,0 +1 @@ +This is taken from https://github.com/tj/n/ diff --git a/helpers/helpers.v2.1.d/vendor/n/n b/helpers/helpers.v2.1.d/vendor/n/n new file mode 100755 index 000000000..86b6a0fa9 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/n/n @@ -0,0 +1,1713 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2155 +# Disabled "Declare and assign separately to avoid masking return values": https://github.com/koalaman/shellcheck/wiki/SC2155 + +# +# log +# + +log() { + printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" +} + +# +# verbose_log +# Can suppress with --quiet. +# Like log but to stderr rather than stdout, so can also be used from "display" routines. +# + +verbose_log() { + if [[ "${SHOW_VERBOSE_LOG}" == "true" ]]; then + >&2 printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" + fi +} + +# +# Exit with the given +# + +abort() { + >&2 printf "\n ${SGR_RED}Error: %s${SGR_RESET}\n\n" "$*" && exit 1 +} + +# +# Synopsis: trace message ... +# Debugging output to stderr, not used in production code. +# + +function trace() { + >&2 printf "trace: %s\n" "$*" +} + +# +# Synopsis: echo_red message ... +# Highlight message in colour (on stdout). +# + +function echo_red() { + printf "${SGR_RED}%s${SGR_RESET}\n" "$*" +} + +# +# Synopsis: n_grep +# grep wrapper to ensure consistent grep options and circumvent aliases. +# + +function n_grep() { + GREP_OPTIONS='' command grep "$@" +} + +# +# Setup and state +# + +VERSION="v9.2.3" + +N_PREFIX="${N_PREFIX-/usr/local}" +N_PREFIX=${N_PREFIX%/} +readonly N_PREFIX + +N_CACHE_PREFIX="${N_CACHE_PREFIX-${N_PREFIX}}" +N_CACHE_PREFIX=${N_CACHE_PREFIX%/} +CACHE_DIR="${N_CACHE_PREFIX}/n/versions" +readonly N_CACHE_PREFIX CACHE_DIR + +N_NODE_MIRROR=${N_NODE_MIRROR:-${NODE_MIRROR:-https://nodejs.org/dist}} +N_NODE_MIRROR=${N_NODE_MIRROR%/} +readonly N_NODE_MIRROR + +N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR:-https://nodejs.org/download} +N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR%/} +readonly N_NODE_DOWNLOAD_MIRROR + +# Using xz instead of gzip is enabled by default, if xz compatibility checks pass. +# User may set N_USE_XZ to 0 to disable, or set to anything else to enable. +# May also be overridden by command line flags. + +# Normalise external values to true/false +if [[ "${N_USE_XZ}" = "0" ]]; then + N_USE_XZ="false" +elif [[ -n "${N_USE_XZ+defined}" ]]; then + N_USE_XZ="true" +fi +# Not setting to readonly. Overriden by CLI flags, and update_xz_settings_for_version. + +N_MAX_REMOTE_MATCHES=${N_MAX_REMOTE_MATCHES:-20} +# modified by update_mirror_settings_for_version +g_mirror_url=${N_NODE_MIRROR} +g_mirror_folder_name="node" + +# Options for curl and wget. +# Defining commands in variables is fraught (https://mywiki.wooledge.org/BashFAQ/050) +# but we can follow the simple case and store arguments in an array. + +GET_SHOWS_PROGRESS="false" +# --location to follow redirects +# --fail to avoid happily downloading error page from web server for 404 et al +# --show-error to show why failed (on stderr) +CURL_OPTIONS=( "--location" "--fail" "--show-error" ) +if [[ -t 1 ]]; then + CURL_OPTIONS+=( "--progress-bar" ) + command -v curl &> /dev/null && GET_SHOWS_PROGRESS="true" +else + CURL_OPTIONS+=( "--silent" ) +fi +WGET_OPTIONS=( "-q" "-O-" ) + +# Legacy support using unprefixed env. No longer documented in README. +if [ -n "$HTTP_USER" ];then + if [ -z "$HTTP_PASSWORD" ]; then + abort "Must specify HTTP_PASSWORD when supplying HTTP_USER" + fi + CURL_OPTIONS+=( "-u $HTTP_USER:$HTTP_PASSWORD" ) + WGET_OPTIONS+=( "--http-password=$HTTP_PASSWORD" + "--http-user=$HTTP_USER" ) +elif [ -n "$HTTP_PASSWORD" ]; then + abort "Must specify HTTP_USER when supplying HTTP_PASSWORD" +fi + +# Set by set_active_node +g_active_node= + +# set by various lookups to allow mixed logging and return value from function, especially for engine and node +g_target_node= + +DOWNLOAD=false # set to opt-out of activate (install), and opt-in to download (run, exec) +ARCH= +SHOW_VERBOSE_LOG="true" +OFFLINE=false + +# ANSI escape codes +# https://en.wikipedia.org/wiki/ANSI_escape_code +# https://no-color.org +# https://bixense.com/clicolors + +USE_COLOR="true" +if [[ -n "${CLICOLOR_FORCE+defined}" && "${CLICOLOR_FORCE}" != "0" ]]; then + USE_COLOR="true" +elif [[ -n "${NO_COLOR+defined}" || "${CLICOLOR}" = "0" || ! -t 1 ]]; then + USE_COLOR="false" +fi +readonly USE_COLOR +# Select Graphic Rendition codes +if [[ "${USE_COLOR}" = "true" ]]; then + # KISS and use codes rather than tput, avoid dealing with missing tput or TERM. + readonly SGR_RESET="\033[0m" + readonly SGR_FAINT="\033[2m" + readonly SGR_RED="\033[31m" + readonly SGR_CYAN="\033[36m" +else + readonly SGR_RESET= + readonly SGR_FAINT= + readonly SGR_RED= + readonly SGR_CYAN= +fi + +# +# set_arch to override $(uname -a) +# + +set_arch() { + if test -n "$1"; then + ARCH="$1" + else + abort "missing -a|--arch value" + fi +} + +# +# Synopsis: set_insecure +# Globals modified: +# - CURL_OPTIONS +# - WGET_OPTIONS +# + +function set_insecure() { + CURL_OPTIONS+=( "--insecure" ) + WGET_OPTIONS+=( "--no-check-certificate" ) +} + +# +# Synposis: display_major_version numeric-version +# +display_major_version() { + local version=$1 + version="${version#v}" + version="${version%%.*}" + echo "${version}" +} + +# +# Synopsis: update_mirror_settings_for_version version +# e.g. means using download mirror and folder is nightly +# Globals modified: +# - g_mirror_url +# - g_mirror_folder_name +# + +function update_mirror_settings_for_version() { + if is_download_folder "$1" ; then + g_mirror_folder_name="$1" + g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" + elif is_download_version "$1"; then + [[ "$1" =~ ^([^/]+)/(.*) ]] + local remote_folder="${BASH_REMATCH[1]}" + g_mirror_folder_name="${remote_folder}" + g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" + fi +} + +# +# Synopsis: update_xz_settings_for_version numeric-version +# Globals modified: +# - N_USE_XZ +# + +function update_xz_settings_for_version() { + # tarballs in xz format were available in later version of iojs, but KISS and only use xz from v4. + if [[ "${N_USE_XZ}" = "true" ]]; then + local major_version="$(display_major_version "$1")" + if [[ "${major_version}" -lt 4 ]]; then + N_USE_XZ="false" + fi + fi +} + +# +# Synopsis: update_arch_settings_for_version numeric-version +# Globals modified: +# - ARCH +# + +function update_arch_settings_for_version() { + local tarball_platform="$(display_tarball_platform)" + if [[ -z "${ARCH}" && "${tarball_platform}" = "darwin-arm64" ]]; then + # First native builds were for v16, but can use x64 in rosetta for older versions. + local major_version="$(display_major_version "$1")" + if [[ "${major_version}" -lt 16 ]]; then + ARCH=x64 + fi + fi +} + +# +# Synopsis: is_lts_codename version +# + +function is_lts_codename() { + # https://github.com/nodejs/Release/blob/master/CODENAMES.md + # e.g. argon, Boron + [[ "$1" =~ ^([Aa]rgon|[Bb]oron|[Cc]arbon|[Dd]ubnium|[Ee]rbium|[Ff]ermium|[Gg]allium|[Hh]ydrogen|[Ii]ron|[Jj]od)$ ]] +} + +# +# Synopsis: is_download_folder version +# + +function is_download_folder() { + # e.g. nightly + [[ "$1" =~ ^(next-nightly|nightly|rc|release|test|v8-canary)$ ]] +} + +# +# Synopsis: is_download_version version +# + +function is_download_version() { + # e.g. nightly/, nightly/latest, nightly/v11 + if [[ "$1" =~ ^([^/]+)/(.*) ]]; then + local remote_folder="${BASH_REMATCH[1]}" + is_download_folder "${remote_folder}" + return + fi + return 2 +} + +# +# Synopsis: is_numeric_version version +# + +function is_numeric_version() { + # e.g. 6, v7.1, 8.11.3 + [[ "$1" =~ ^[v]{0,1}[0-9]+(\.[0-9]+){0,2}$ ]] +} + +# +# Synopsis: is_exact_numeric_version version +# + +function is_exact_numeric_version() { + # e.g. 6, v7.1, 8.11.3 + [[ "$1" =~ ^[v]{0,1}[0-9]+\.[0-9]+\.[0-9]+$ ]] +} + +# +# Synopsis: is_node_support_version version +# Reference: https://github.com/nodejs/package-maintenance/issues/236#issue-474783582 +# + +function is_node_support_version() { + [[ "$1" =~ ^(active|lts_active|lts_latest|lts|current|supported)$ ]] +} + +# +# Synopsis: display_latest_node_support_alias version +# Map aliases onto existing n aliases, current and lts +# + +function display_latest_node_support_alias() { + case "$1" in + "active") printf "current" ;; + "lts_active") printf "lts" ;; + "lts_latest") printf "lts" ;; + "lts") printf "lts" ;; + "current") printf "current" ;; + "supported") printf "current" ;; + *) printf "unexpected-version" + esac +} + +# +# Functions used when showing versions installed +# + +enter_fullscreen() { + # Set cursor to be invisible + tput civis 2> /dev/null + # Save screen contents + tput smcup 2> /dev/null + stty -echo +} + +leave_fullscreen() { + # Set cursor to normal + tput cnorm 2> /dev/null + # Restore screen contents + tput rmcup 2> /dev/null + stty echo +} + +handle_sigint() { + leave_fullscreen + S="$?" + kill 0 + exit $S +} + +handle_sigtstp() { + leave_fullscreen + kill -s SIGSTOP $$ +} + +# +# Output usage information. +# + +display_help() { + cat <<-EOF + +Usage: n [options] [COMMAND] [args] + +Commands: + + n Display downloaded Node.js versions and install selection + n latest Install the latest Node.js release (downloading if necessary) + n lts Install the latest LTS Node.js release (downloading if necessary) + n Install Node.js (downloading if necessary) + n install Install Node.js (downloading if necessary) + n run [args ...] Execute downloaded Node.js with [args ...] + n which Output path for downloaded node + n exec [args...] Execute command with modified PATH, so downloaded node and npm first + n rm Remove the given downloaded version(s) + n prune Remove all downloaded versions except the installed version + n --latest Output the latest Node.js version available + n --lts Output the latest LTS Node.js version available + n ls Output downloaded versions + n ls-remote [version] Output matching versions available for download + n uninstall Remove the installed Node.js + +Options: + + -V, --version Output version of n + -h, --help Display help information + -p, --preserve Preserve npm and npx during install of Node.js + -q, --quiet Disable curl output. Disable log messages processing "auto" and "engine" labels. + -d, --download Download if necessary, and don't make active + -a, --arch Override system architecture + --offline Resolve target version against cached downloads instead of internet lookup + --all ls-remote displays all matches instead of last 20 + --insecure Turn off certificate checking for https requests (may be needed from behind a proxy server) + --use-xz/--no-use-xz Override automatic detection of xz support and enable/disable use of xz compressed node downloads. + +Aliases: + + install: i + latest: current + ls: list + lsr: ls-remote + lts: stable + rm: - + run: use, as + which: bin + +Versions: + + Numeric version numbers can be complete or incomplete, with an optional leading 'v'. + Versions can also be specified by label, or codename, + and other downloadable releases by / + + 4.9.1, 8, v6.1 Numeric versions + lts Newest Long Term Support official release + latest, current Newest official release + auto Read version from file: .n-node-version, .node-version, .nvmrc, or package.json + engine Read version from package.json + boron, carbon Codenames for release streams + lts_latest Node.js support aliases + + and nightly, rc/10 et al + +EOF +} + +err_no_installed_print_help() { + display_help + abort "no downloaded versions yet, see above help for commands" +} + +# +# Synopsis: next_version_installed selected_version +# Output version after selected (which may be blank under some circumstances). +# + +function next_version_installed() { + display_cache_versions | n_grep "$1" -A 1 | tail -n 1 +} + +# +# Synopsis: prev_version_installed selected_version +# Output version before selected (which may be blank under some circumstances). +# + +function prev_version_installed() { + display_cache_versions | n_grep "$1" -B 1 | head -n 1 +} + +# +# Output n version. +# + +display_n_version() { + echo "$VERSION" && exit 0 +} + +# +# Synopsis: set_active_node +# Checks cached downloads for a binary matching the active node. +# Globals modified: +# - g_active_node +# + +function set_active_node() { + g_active_node= + local node_path="$(command -v node)" + if [[ -x "${node_path}" ]]; then + local installed_version=$(node --version) + installed_version=${installed_version#v} + for dir in "${CACHE_DIR}"/*/ ; do + local folder_name="${dir%/}" + folder_name="${folder_name##*/}" + if diff &> /dev/null \ + "${CACHE_DIR}/${folder_name}/${installed_version}/bin/node" \ + "${node_path}" ; then + g_active_node="${folder_name}/${installed_version}" + break + fi + done + fi +} + +# +# Display sorted versions directories paths. +# + +display_versions_paths() { + find "$CACHE_DIR" -maxdepth 2 -type d \ + | sed 's|'"$CACHE_DIR"'/||g' \ + | n_grep -E "/[0-9]+\.[0-9]+\.[0-9]+" \ + | sed 's|/|.|' \ + | sort -k 1,1 -k 2,2n -k 3,3n -k 4,4n -t . \ + | sed 's|\.|/|' +} + +# +# Display installed versions with +# + +display_versions_with_selected() { + local selected="$1" + echo + for version in $(display_versions_paths); do + if test "$version" = "$selected"; then + printf " ${SGR_CYAN}ο${SGR_RESET} %s\n" "$version" + else + printf " ${SGR_FAINT}%s${SGR_RESET}\n" "$version" + fi + done + echo + printf "Use up/down arrow keys to select a version, return key to install, d to delete, q to quit" +} + +# +# Synopsis: display_cache_versions +# + +function display_cache_versions() { + for folder_and_version in $(display_versions_paths); do + echo "${folder_and_version}" + done +} + +# +# Display current node --version and others installed. +# + +menu_select_cache_versions() { + enter_fullscreen + set_active_node + local selected="${g_active_node}" + + clear + display_versions_with_selected "${selected}" + + trap handle_sigint INT + trap handle_sigtstp SIGTSTP + + ESCAPE_SEQ=$'\033' + UP=$'A' + DOWN=$'B' + CTRL_P=$'\020' + CTRL_N=$'\016' + + while true; do + read -rsn 1 key + case "$key" in + "$ESCAPE_SEQ") + # Handle ESC sequences followed by other characters, i.e. arrow keys + read -rsn 1 -t 1 tmp + # See "[" if terminal in normal mode, and "0" in application mode + if [[ "$tmp" == "[" || "$tmp" == "O" ]]; then + read -rsn 1 -t 1 arrow + case "$arrow" in + "$UP") + clear + selected="$(prev_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + "$DOWN") + clear + selected="$(next_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + esac + fi + ;; + "d") + if [[ -n "${selected}" ]]; then + clear + # Note: prev/next is constrained to min/max + local after_delete_selection="$(next_version_installed "${selected}")" + if [[ "${after_delete_selection}" == "${selected}" ]]; then + after_delete_selection="$(prev_version_installed "${selected}")" + fi + remove_versions "${selected}" + + if [[ "${after_delete_selection}" == "${selected}" ]]; then + clear + leave_fullscreen + echo "All downloaded versions have been deleted from cache." + exit + fi + + selected="${after_delete_selection}" + display_versions_with_selected "${selected}" + fi + ;; + # Vim or Emacs 'up' key + "k"|"$CTRL_P") + clear + selected="$(prev_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + # Vim or Emacs 'down' key + "j"|"$CTRL_N") + clear + selected="$(next_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + "q") + clear + leave_fullscreen + exit + ;; + "") + # enter key returns empty string + leave_fullscreen + [[ -n "${selected}" ]] && activate "${selected}" + exit + ;; + esac + done +} + +# +# Move up a line and erase. +# + +erase_line() { + printf "\033[1A\033[2K" +} + +# +# Disable PaX mprotect for +# + +disable_pax_mprotect() { + test -z "$1" && abort "binary required" + local binary="$1" + + # try to disable mprotect via XATTR_PAX header + local PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl-ng 2>&1)" + local PAXCTL_ERROR=1 + if [ -x "$PAXCTL" ]; then + $PAXCTL -l && $PAXCTL -m "$binary" >/dev/null 2>&1 + PAXCTL_ERROR="$?" + fi + + # try to disable mprotect via PT_PAX header + if [ "$PAXCTL_ERROR" != 0 ]; then + PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl 2>&1)" + if [ -x "$PAXCTL" ]; then + $PAXCTL -Cm "$binary" >/dev/null 2>&1 + fi + fi +} + +# +# clean_copy_folder +# + +clean_copy_folder() { + local source="$1" + local target="$2" + if [[ -d "${source}" ]]; then + rm -rf "${target}" + cp -fR "${source}" "${target}" + fi +} + +# +# Activate +# + +activate() { + local version="$1" + local dir="$CACHE_DIR/$version" + local original_node="$(command -v node)" + local installed_node="${N_PREFIX}/bin/node" + log "copying" "$version" + + + # Ideally we would just copy from cache to N_PREFIX, but there are some complications + # - various linux versions use symlinks for folders in /usr/local and also error when copy folder onto symlink + # - we have used cp for years, so keep using it for backwards compatibility (instead of say rsync) + # - we allow preserving npm + # - we want to be somewhat robust to changes in tarball contents, so use find instead of hard-code expected subfolders + # + # This code was purist and concise for a long time. + # Now twice as much code, but using same code path for all uses, and supporting more setups. + + # Copy lib before bin so symlink targets exist. + # lib + mkdir -p "$N_PREFIX/lib" + # Copy everything except node_modules. + find "$dir/lib" -mindepth 1 -maxdepth 1 \! -name node_modules -exec cp -fR "{}" "$N_PREFIX/lib" \; + if [[ -z "${N_PRESERVE_NPM}" ]]; then + mkdir -p "$N_PREFIX/lib/node_modules" + # Copy just npm, skipping possible added global modules after download. Clean copy to avoid version change problems. + clean_copy_folder "$dir/lib/node_modules/npm" "$N_PREFIX/lib/node_modules/npm" + fi + # Takes same steps for corepack (experimental in node 16.9.0) as for npm, to avoid version problems. + if [[ -e "$dir/lib/node_modules/corepack" && -z "${N_PRESERVE_COREPACK}" ]]; then + mkdir -p "$N_PREFIX/lib/node_modules" + clean_copy_folder "$dir/lib/node_modules/corepack" "$N_PREFIX/lib/node_modules/corepack" + fi + + # bin + mkdir -p "$N_PREFIX/bin" + # Remove old node to avoid potential problems with firewall getting confused on Darwin by overwrite. + rm -f "$N_PREFIX/bin/node" + # Copy bin items by hand, in case user has installed global npm modules into cache. + cp -f "$dir/bin/node" "$N_PREFIX/bin" + [[ -e "$dir/bin/node-waf" ]] && cp -f "$dir/bin/node-waf" "$N_PREFIX/bin" # v0.8.x + if [[ -z "${N_PRESERVE_COREPACK}" ]]; then + [[ -e "$dir/bin/corepack" ]] && cp -fR "$dir/bin/corepack" "$N_PREFIX/bin" # from 16.9.0 + fi + if [[ -z "${N_PRESERVE_NPM}" ]]; then + [[ -e "$dir/bin/npm" ]] && cp -fR "$dir/bin/npm" "$N_PREFIX/bin" + [[ -e "$dir/bin/npx" ]] && cp -fR "$dir/bin/npx" "$N_PREFIX/bin" + fi + + # include + mkdir -p "$N_PREFIX/include" + find "$dir/include" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/include" \; + + # share + mkdir -p "$N_PREFIX/share" + # Copy everything except man, at it is a symlink on some Linux (e.g. archlinux). + find "$dir/share" -mindepth 1 -maxdepth 1 \! -name man -exec cp -fR "{}" "$N_PREFIX/share" \; + mkdir -p "$N_PREFIX/share/man" + find "$dir/share/man" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/share/man" \; + + disable_pax_mprotect "${installed_node}" + + local active_node="$(command -v node)" + if [[ -e "${active_node}" && -e "${installed_node}" && "${active_node}" != "${installed_node}" ]]; then + # Installed and active are different which might be a PATH problem. List both to give user some clues. + log "installed" "$("${installed_node}" --version) to ${installed_node}" + log "active" "$("${active_node}" --version) at ${active_node}" + else + local npm_version_str="" + local installed_npm="${N_PREFIX}/bin/npm" + local active_npm="$(command -v npm)" + if [[ -z "${N_PRESERVE_NPM}" && -e "${active_npm}" && -e "${installed_npm}" && "${active_npm}" = "${installed_npm}" ]]; then + npm_version_str=" (with npm $(npm --version))" + fi + + log "installed" "$("${installed_node}" --version)${npm_version_str}" + + # Extra tips for changed location. + if [[ -e "${active_node}" && -e "${original_node}" && "${active_node}" != "${original_node}" ]]; then + printf '\nNote: the node command changed location and the old location may be remembered in your current shell.\n' + log old "${original_node}" + log new "${active_node}" + printf 'If "node --version" shows the old version then start a new shell, or reset the location hash with:\nhash -r (for bash, zsh, ash, dash, and ksh)\nrehash (for csh and tcsh)\n' + fi + fi +} + +# +# Install +# + +install() { + [[ -z "$1" ]] && abort "version required" + local version + get_latest_resolved_version "$1" || return 2 + version="${g_target_node}" + [[ -n "${version}" ]] || abort "no version found for '$1'" + update_mirror_settings_for_version "$1" + update_xz_settings_for_version "${version}" + update_arch_settings_for_version "${version}" + + local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + + # Note: decompression flags ignored with default Darwin tar which autodetects. + if test "$N_USE_XZ" = "true"; then + local tarflag="-Jx" + else + local tarflag="-zx" + fi + + if test -d "$dir"; then + if [[ ! -e "$dir/n.lock" ]] ; then + if [[ "$DOWNLOAD" == "false" ]] ; then + activate "${g_mirror_folder_name}/${version}" + fi + exit + fi + fi + if [[ "$OFFLINE" == "true" ]]; then + abort "version unavailable offline" + fi + + log installing "${g_mirror_folder_name}-v$version" + + local url="$(tarball_url "$version")" + is_ok "${url}" || abort "download preflight failed for '$version' (${url})" + + log mkdir "$dir" + mkdir -p "$dir" || abort "sudo required (or change ownership, or define N_PREFIX)" + touch "$dir/n.lock" + + cd "${dir}" || abort "Failed to cd to ${dir}" + + log fetch "$url" + do_get "${url}" | tar "$tarflag" --strip-components=1 --no-same-owner -f - + pipe_results=( "${PIPESTATUS[@]}" ) + if [[ "${pipe_results[0]}" -ne 0 ]]; then + abort "failed to download archive for $version" + fi + if [[ "${pipe_results[1]}" -ne 0 ]]; then + abort "failed to extract archive for $version" + fi + [ "$GET_SHOWS_PROGRESS" = "true" ] && erase_line + rm -f "$dir/n.lock" + + disable_pax_mprotect bin/node + + if [[ "$DOWNLOAD" == "false" ]]; then + activate "${g_mirror_folder_name}/$version" + fi +} + +# +# Be more silent. +# + +set_quiet() { + SHOW_VERBOSE_LOG="false" + command -v curl > /dev/null && CURL_OPTIONS+=( "--silent" ) && GET_SHOWS_PROGRESS="false" +} + +# +# Synopsis: do_get [option...] url +# Call curl or wget with combination of global and passed options. +# + +function do_get() { + if command -v curl &> /dev/null; then + curl "${CURL_OPTIONS[@]}" "$@" + elif command -v wget &> /dev/null; then + wget "${WGET_OPTIONS[@]}" "$@" + else + abort "curl or wget command required" + fi +} + +# +# Synopsis: do_get_index [option...] url +# Call curl or wget with combination of global and passed options, +# with options tweaked to be more suitable for getting index. +# + +function do_get_index() { + if command -v curl &> /dev/null; then + # --silent to suppress progress et al + curl --silent "${CURL_OPTIONS[@]}" "$@" + elif command -v wget &> /dev/null; then + wget "${WGET_OPTIONS[@]}" "$@" + else + abort "curl or wget command required" + fi +} + +# +# Synopsis: remove_versions version ... +# + +function remove_versions() { + [[ -z "$1" ]] && abort "version(s) required" + while [[ $# -ne 0 ]]; do + local version + get_latest_resolved_version "$1" || break + version="${g_target_node}" + if [[ -n "${version}" ]]; then + update_mirror_settings_for_version "$1" + local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + if [[ -s "${dir}" ]]; then + rm -rf "${dir}" + else + echo "$1 (${version}) not in downloads cache" + fi + else + echo "No version found for '$1'" + fi + shift + done +} + +# +# Synopsis: prune_cache +# + +function prune_cache() { + set_active_node + + for folder_and_version in $(display_versions_paths); do + if [[ "${folder_and_version}" != "${g_active_node}" ]]; then + echo "${folder_and_version}" + rm -rf "${CACHE_DIR:?}/${folder_and_version}" + fi + done +} + +# +# Synopsis: find_cached_version version +# Finds cache directory for resolved version. +# Globals modified: +# - g_cached_version + +function find_cached_version() { + [[ -z "$1" ]] && abort "version required" + local version + get_latest_resolved_version "$1" || exit 1 + version="${g_target_node}" + [[ -n "${version}" ]] || abort "no version found for '$1'" + + update_mirror_settings_for_version "$1" + g_cached_version="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + if [[ ! -d "${g_cached_version}" && "${DOWNLOAD}" == "true" ]]; then + (install "${version}") + fi + [[ -d "${g_cached_version}" ]] || abort "'$1' (${version}) not in downloads cache" +} + + +# +# Synopsis: display_bin_path_for_version version +# + +function display_bin_path_for_version() { + find_cached_version "$1" + echo "${g_cached_version}/bin/node" +} + +# +# Synopsis: run_with_version version [args...] +# Run the given of node with [args ..] +# + +function run_with_version() { + find_cached_version "$1" + shift # remove version from parameters + exec "${g_cached_version}/bin/node" "$@" +} + +# +# Synopsis: exec_with_version command [args...] +# Modify the path to include and execute command. +# + +function exec_with_version() { + find_cached_version "$1" + shift # remove version from parameters + PATH="${g_cached_version}/bin:$PATH" exec "$@" +} + +# +# Synopsis: is_ok url +# Check the HEAD response of . +# + +function is_ok() { + # Note: both curl and wget can follow redirects, as present on some mirrors (e.g. https://npm.taobao.org/mirrors/node). + # The output is complicated with redirects, so keep it simple and use command status rather than parse output. + if command -v curl &> /dev/null; then + do_get --silent --head "$1" > /dev/null || return 1 + else + do_get --spider "$1" > /dev/null || return 1 + fi +} + +# +# Synopsis: can_use_xz +# Test system to see if xz decompression is supported by tar. +# + +function can_use_xz() { + # Be conservative and only enable if xz is likely to work. Unfortunately we can't directly query tar itself. + # For research, see https://github.com/shadowspawn/nvh/issues/8 + local uname_s="$(uname -s)" + if [[ "${uname_s}" = "Linux" ]] && command -v xz &> /dev/null ; then + # tar on linux is likely to support xz if it is available as a command + return 0 + elif [[ "${uname_s}" = "Darwin" ]]; then + local macos_version="$(sw_vers -productVersion)" + local macos_major_version="$(echo "${macos_version}" | cut -d '.' -f 1)" + local macos_minor_version="$(echo "${macos_version}" | cut -d '.' -f 2)" + if [[ "${macos_major_version}" -gt 10 || "${macos_minor_version}" -gt 8 ]]; then + # tar on recent Darwin has xz support built-in + return 0 + fi + fi + return 2 # not supported +} + +# +# Synopsis: display_tarball_platform +# + +function display_tarball_platform() { + # https://en.wikipedia.org/wiki/Uname + + local os="unexpected_os" + local uname_a="$(uname -a)" + case "${uname_a}" in + Linux*) os="linux" ;; + Darwin*) os="darwin" ;; + SunOS*) os="sunos" ;; + AIX*) os="aix" ;; + CYGWIN*) >&2 echo_red "Cygwin is not supported by n" ;; + MINGW*) >&2 echo_red "Git BASH (MSYS) is not supported by n" ;; + esac + + local arch="unexpected_arch" + local uname_m="$(uname -m)" + case "${uname_m}" in + x86_64) arch=x64 ;; + i386 | i686) arch="x86" ;; + aarch64) arch=arm64 ;; + armv8l) arch=arm64 ;; # armv8l probably supports arm64, and there is no specific armv8l build so give it a go + *) + # e.g. armv6l, armv7l, arm64 + arch="${uname_m}" + ;; + esac + # Override from command line, or version specific adjustment. + [ -n "$ARCH" ] && arch="$ARCH" + + echo "${os}-${arch}" +} + +# +# Synopsis: display_compatible_file_field +# display for current platform, as per field in index.tab, which is different than actual download +# + +function display_compatible_file_field { + local compatible_file_field="$(display_tarball_platform)" + if [[ -z "${ARCH}" && "${compatible_file_field}" = "darwin-arm64" ]]; then + # Look for arm64 for native but also x64 for older versions which can run in rosetta. + # (Downside is will get an install error if install version above 16 with x64 and not arm64.) + compatible_file_field="osx-arm64-tar|osx-x64-tar" + elif [[ "${compatible_file_field}" =~ darwin-(.*) ]]; then + compatible_file_field="osx-${BASH_REMATCH[1]}-tar" + fi + echo "${compatible_file_field}" +} + +# +# Synopsis: tarball_url version +# + +function tarball_url() { + local version="$1" + local ext=gz + [ "$N_USE_XZ" = "true" ] && ext="xz" + echo "${g_mirror_url}/v${version}/node-v${version}-$(display_tarball_platform).tar.${ext}" +} + +# +# Synopsis: get_file_node_version filename +# Sets g_target_node +# + +function get_file_node_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + # read returns a non-zero status but does still work if there is no line ending + local version + <"${filepath}" read -r version + # trim possible trailing \d from a Windows created file + version="${version%%[[:space:]]}" + verbose_log "read" "${version}" + g_target_node="${version}" +} + +# +# Synopsis: get_package_engine_version\ +# Sets g_target_node +# + +function get_package_engine_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + command -v node &> /dev/null || abort "an active version of node is required to read 'engines' from package.json" + local range + range="$(node -e "package = require('${filepath}'); if (package && package.engines && package.engines.node) console.log(package.engines.node)")" + verbose_log "read" "${range}" + [[ -n "${range}" ]] || return 2 + if [[ "*" == "${range}" ]]; then + verbose_log "target" "current" + g_target_node="current" + return + fi + + local version + if [[ "${range}" =~ ^([>~^=]|\>\=)?v?([0-9]+(\.[0-9]+){0,2})(.[xX*])?$ ]]; then + local operator="${BASH_REMATCH[1]}" + version="${BASH_REMATCH[2]}" + case "${operator}" in + '' | =) ;; + \> | \>=) version="current" ;; + \~) [[ "${version}" =~ ^([0-9]+\.[0-9]+)\.[0-9]+$ ]] && version="${BASH_REMATCH[1]}" ;; + ^) [[ "${version}" =~ ^([0-9]+) ]] && version="${BASH_REMATCH[1]}" ;; + esac + verbose_log "target" "${version}" + else + command -v npx &> /dev/null || abort "an active version of npx is required to use complex 'engine' ranges from package.json" + [[ "$OFFLINE" != "true" ]] || abort "offline: an internet connection is required for looking up complex 'engine' ranges from package.json" + verbose_log "resolving" "${range}" + local version_per_line="$(n lsr --all)" + local versions_one_line=$(echo "${version_per_line}" | tr '\n' ' ') + # Using semver@7 so works with older versions of node. + # shellcheck disable=SC2086 + version=$(npm_config_yes=true npx --quiet semver@7 -r "${range}" ${versions_one_line} | tail -n 1) + fi + g_target_node="${version}" +} + +# +# Synopsis: get_nvmrc_version +# Sets g_target_node +# + +function get_nvmrc_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + local version + <"${filepath}" read -r version + verbose_log "read" "${version}" + # Translate from nvm aliases + case "${version}" in + lts/\*) version="lts" ;; + lts/*) version="${version:4}" ;; + node) version="current" ;; + *) ;; + esac + g_target_node="${version}" +} + +# +# Synopsis: get_engine_version [error-message] +# Sets g_target_node +# + +function get_engine_version() { + g_target_node= + local error_message="${1-package.json not found}" + local parent + parent="${PWD}" + while [[ -n "${parent}" ]]; do + if [[ -e "${parent}/package.json" ]]; then + get_package_engine_version "${parent}/package.json" + else + parent=${parent%/*} + continue + fi + break + done + [[ -n "${parent}" ]] || abort "${error_message}" + [[ -n "${g_target_node}" ]] || abort "did not find supported version of node in 'engines' field of package.json" +} + +# +# Synopsis: get_auto_version +# Sets g_target_node +# + +function get_auto_version() { + g_target_node= + # Search for a version control file first + local parent + parent="${PWD}" + while [[ -n "${parent}" ]]; do + if [[ -e "${parent}/.n-node-version" ]]; then + get_file_node_version "${parent}/.n-node-version" + elif [[ -e "${parent}/.node-version" ]]; then + get_file_node_version "${parent}/.node-version" + elif [[ -e "${parent}/.nvmrc" ]]; then + get_nvmrc_version "${parent}/.nvmrc" + else + parent=${parent%/*} + continue + fi + break + done + # Fallback to package.json + [[ -n "${parent}" ]] || get_engine_version "no file found for auto version (.n-node-version, .node-version, .nvmrc, or package.json)" + [[ -n "${g_target_node}" ]] || abort "file found for auto did not contain target version of node" +} + +# +# Synopsis: get_latest_resolved_version version +# Sets g_target_node +# + +function get_latest_resolved_version() { + g_target_node= + local version=${1} + simple_version=${version#node/} # Only place supporting node/ [sic] + if is_exact_numeric_version "${simple_version}"; then + # Just numbers, already resolved, no need to lookup first. + simple_version="${simple_version#v}" + g_target_node="${simple_version}" + elif [[ "$OFFLINE" == "true" ]]; then + g_target_node=$(display_local_versions "${version}") + else + # Complicated recognising exact version, KISS and lookup. + g_target_node=$(N_MAX_REMOTE_MATCHES=1 display_remote_versions "$version") + fi +} + +# +# Synopsis: display_remote_index +# index.tab reference: https://github.com/nodejs/nodejs-dist-indexer +# Index fields are: version date files npm v8 uv zlib openssl modules lts security +# KISS and just return fields we currently care about: version files lts +# + +display_remote_index() { + local index_url="${g_mirror_url}/index.tab" + # tail to remove header line + do_get_index "${index_url}" | tail -n +2 | cut -f 1,3,10 + if [[ "${PIPESTATUS[0]}" -ne 0 ]]; then + # Reminder: abort will only exit subshell, but consistent error display + abort "failed to download version index (${index_url})" + fi +} + +# +# Synopsis: display_match_limit limit +# + +function display_match_limit(){ + if [[ "$1" -gt 1 && "$1" -lt 32000 ]]; then + echo "Listing remote... Displaying $1 matches (use --all to see all)." + fi +} + +# +# Synopsis: display_local_versions version +# + +function display_local_versions() { + local version="$1" + local match='.' + verbose_log "offline" "matching cached versions" + + # Transform some labels before processing further. + if is_node_support_version "${version}"; then + version="$(display_latest_node_support_alias "${version}")" + match_count=1 + elif [[ "${version}" = "auto" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_auto_version || return 2 + version="${g_target_node}" + elif [[ "${version}" = "engine" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_engine_version || return 2 + version="${g_target_node}" + fi + + if [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match='^node/.' + elif is_exact_numeric_version "${version}"; then + # Quote any dots in version so they are literal for expression + match="^node/${version//\./\.}" + elif is_numeric_version "${version}"; then + version="${version#v}" + # Quote any dots in version so they are literal for expression + match="${version//\./\.}" + # Avoid 1.2 matching 1.23 + match="^node/${match}[^0-9]" + # elif is_lts_codename "${version}"; then + # see if demand + elif is_download_folder "${version}"; then + match="^${version}/" + # elif is_download_version "${version}"; then + # see if demand + else + abort "invalid version '$1' for offline matching" + fi + + display_versions_paths \ + | n_grep -E "${match}" \ + | tail -n 1 \ + | sed 's|node/||' +} + +# +# Synopsis: display_remote_versions version +# + +function display_remote_versions() { + local version="$1" + update_mirror_settings_for_version "${version}" + local match='.' + local match_count="${N_MAX_REMOTE_MATCHES}" + + # Transform some labels before processing further. + if is_node_support_version "${version}"; then + version="$(display_latest_node_support_alias "${version}")" + match_count=1 + elif [[ "${version}" = "auto" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_auto_version || return 2 + version="${g_target_node}" + elif [[ "${version}" = "engine" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_engine_version || return 2 + version="${g_target_node}" + fi + + if [[ -z "${version}" ]]; then + match='.' + elif [[ "${version}" = "lts" || "${version}" = "stable" ]]; then + match_count=1 + # Codename is last field, first one with a name is newest lts + match="${TAB_CHAR}[a-zA-Z]+\$" + elif [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match_count=1 + match='.' + elif is_numeric_version "${version}"; then + version="v${version#v}" + # Avoid restriction message if exact version + is_exact_numeric_version "${version}" && match_count=1 + # Quote any dots in version so they are literal for expression + match="${version//\./\.}" + # Avoid 1.2 matching 1.23 + match="^${match}[^0-9]" + elif is_lts_codename "${version}"; then + # Capitalise (could alternatively make grep case insensitive) + codename="$(echo "${version:0:1}" | tr '[:lower:]' '[:upper:]')${version:1}" + # Codename is last field + match="${TAB_CHAR}${codename}\$" + elif is_download_folder "${version}"; then + match='.' + elif is_download_version "${version}"; then + version="${version#"${g_mirror_folder_name}"/}" + if [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match_count=1 + match='.' + else + version="v${version#v}" + match="${version//\./\.}" + match="^${match}" # prefix + if is_numeric_version "${version}"; then + # Exact numeric match + match="${match}[^0-9]" + fi + fi + else + abort "invalid version '$1'" + fi + display_match_limit "${match_count}" + + # Implementation notes: + # - using awk rather than head so do not close pipe early on curl + # - restrict search to compatible files as not always available, or not at same time + # - return status of curl command (i.e. PIPESTATUS[0]) + display_remote_index \ + | n_grep -E "$(display_compatible_file_field)" \ + | n_grep -E "${match}" \ + | awk "NR<=${match_count}" \ + | cut -f 1 \ + | n_grep -E -o '[^v].*' + return "${PIPESTATUS[0]}" +} + +# +# Synopsis: delete_with_echo target +# + +function delete_with_echo() { + if [[ -e "$1" ]]; then + echo "$1" + rm -rf "$1" + fi +} + +# +# Synopsis: uninstall_installed +# Uninstall the installed node and npm (leaving alone the cache), +# so undo install, and may expose possible system installed versions. +# + +uninstall_installed() { + # npm: https://docs.npmjs.com/misc/removing-npm + # rm -rf /usr/local/{lib/node{,/.npm,_modules},bin,share/man}/npm* + # node: https://stackabuse.com/how-to-uninstall-node-js-from-mac-osx/ + # Doing it by hand rather than scanning cache, so still works if cache deleted first. + # This covers tarballs for at least node 4 through 10. + + while true; do + read -r -p "Do you wish to delete node and npm from ${N_PREFIX}? " yn + case $yn in + [Yy]* ) break ;; + [Nn]* ) exit ;; + * ) echo "Please answer yes or no.";; + esac + done + + echo "" + echo "Uninstalling node and npm" + delete_with_echo "${N_PREFIX}/bin/node" + delete_with_echo "${N_PREFIX}/bin/npm" + delete_with_echo "${N_PREFIX}/bin/npx" + delete_with_echo "${N_PREFIX}/bin/corepack" + delete_with_echo "${N_PREFIX}/include/node" + delete_with_echo "${N_PREFIX}/lib/dtrace/node.d" + delete_with_echo "${N_PREFIX}/lib/node_modules/npm" + delete_with_echo "${N_PREFIX}/lib/node_modules/corepack" + delete_with_echo "${N_PREFIX}/share/doc/node" + delete_with_echo "${N_PREFIX}/share/man/man1/node.1" + delete_with_echo "${N_PREFIX}/share/systemtap/tapset/node.stp" +} + +# +# Synopsis: show_permission_suggestions +# + +function show_permission_suggestions() { + echo "Suggestions:" + echo "- run n with sudo, or" + if [[ "${N_CACHE_PREFIX}" == "${N_PREFIX}" ]]; then + echo "- define N_PREFIX to a writeable location, or" + else + echo "- define N_PREFIX and N_CACHE_PREFIX to writeable locations, or" + fi +} + +# +# Synopsis: show_diagnostics +# Show environment and check for common problems. +# + +function show_diagnostics() { + echo "This information is to help you diagnose issues, and useful when reporting an issue." + echo "Note: some output may contain passwords. Redact before sharing." + + printf "\n\nCOMMAND LOCATIONS AND VERSIONS\n" + + printf "\nbash\n" + command -v bash && bash --version + + printf "\nn\n" + command -v n && n --version + + printf "\nnode\n" + if command -v node &> /dev/null; then + command -v node && node --version + node -e 'if (process.versions.v8) console.log("JavaScript engine: v8");' + + printf "\nnpm\n" + command -v npm && npm --version + fi + + printf "\ntar\n" + if command -v tar &> /dev/null; then + command -v tar && tar --version + else + echo_red "tar not found. Needed for extracting downloads." + fi + + printf "\ncurl or wget\n" + if command -v curl &> /dev/null; then + command -v curl && curl --version + elif command -v wget &> /dev/null; then + command -v wget && wget --version + else + echo_red "Neither curl nor wget found. Need one of them for downloads." + fi + + printf "\nuname\n" + uname -a + + printf "\n\nSETTINGS\n" + + printf "\nn\n" + echo "node mirror: ${N_NODE_MIRROR}" + echo "node downloads mirror: ${N_NODE_DOWNLOAD_MIRROR}" + echo "install destination: ${N_PREFIX}" + [[ -n "${N_PREFIX}" ]] && echo "PATH: ${PATH}" + echo "ls-remote max matches: ${N_MAX_REMOTE_MATCHES}" + [[ -n "${N_PRESERVE_NPM}" ]] && echo "installs preserve npm by default" + [[ -n "${N_PRESERVE_COREPACK}" ]] && echo "installs preserve corepack by default" + + printf "\nProxy\n" + # disable "var is referenced but not assigned": https://github.com/koalaman/shellcheck/wiki/SC2154 + # shellcheck disable=SC2154 + [[ -n "${http_proxy}" ]] && echo "http_proxy: ${http_proxy}" + # shellcheck disable=SC2154 + [[ -n "${https_proxy}" ]] && echo "https_proxy: ${https_proxy}" + if command -v curl &> /dev/null; then + # curl supports lower case and upper case! + # shellcheck disable=SC2154 + [[ -n "${all_proxy}" ]] && echo "all_proxy: ${all_proxy}" + [[ -n "${ALL_PROXY}" ]] && echo "ALL_PROXY: ${ALL_PROXY}" + [[ -n "${HTTP_PROXY}" ]] && echo "HTTP_PROXY: ${HTTP_PROXY}" + [[ -n "${HTTPS_PROXY}" ]] && echo "HTTPS_PROXY: ${HTTPS_PROXY}" + if [[ -e "${CURL_HOME}/.curlrc" ]]; then + echo "have \$CURL_HOME/.curlrc" + elif [[ -e "${HOME}/.curlrc" ]]; then + echo "have \$HOME/.curlrc" + fi + elif command -v wget &> /dev/null; then + if [[ -e "${WGETRC}" ]]; then + echo "have \$WGETRC" + elif [[ -e "${HOME}/.wgetrc" ]]; then + echo "have \$HOME/.wgetrc" + fi + fi + + printf "\n\nCHECKS\n" + + printf "\nChecking n install destination is in PATH...\n" + local install_bin="${N_PREFIX}/bin" + local path_wth_guards=":${PATH}:" + if [[ "${path_wth_guards}" =~ :${install_bin}/?: ]]; then + printf "good\n" + else + echo_red "'${install_bin}' is not in PATH" + fi + if command -v node &> /dev/null; then + printf "\nChecking n install destination priority in PATH...\n" + local node_dir="$(dirname "$(command -v node)")" + + local index=0 + local path_entry + local path_entries + local install_bin_index=0 + local node_index=999 + IFS=':' read -ra path_entries <<< "${PATH}" + for path_entry in "${path_entries[@]}"; do + (( index++ )) + [[ "${path_entry}" =~ ^${node_dir}/?$ ]] && node_index="${index}" + [[ "${path_entry}" =~ ^${install_bin}/?$ ]] && install_bin_index="${index}" + done + if [[ "${node_index}" -lt "${install_bin_index}" ]]; then + echo_red "There is a version of node installed which will be found in PATH before the n installed version." + else + printf "good\n" + fi + fi + + # Check npm too. Simpler check than for PATH and node, more like the runtime logging for active/installed node. + if [[ -z "${N_PRESERVE_NPM}" ]]; then + printf "\nChecking npm install destination...\n" + local installed_npm="${N_PREFIX}/bin/npm" + local active_npm="$(command -v npm)" + if [[ -e "${active_npm}" && -e "${installed_npm}" && "${active_npm}" != "${installed_npm}" ]]; then + echo_red "There is an active version of npm shadowing the version installed by n. Check order of entries in PATH." + log "installed" "${installed_npm}" + log "active" "${active_npm}" + else + printf "good\n" + fi + fi + + printf "\nChecking prefix folders...\n" + if [[ ! -e "${N_PREFIX}" ]]; then + echo "Folder does not exist: ${N_PREFIX}" + echo "- This folder will be created when you do an install." + fi + if [[ "${N_PREFIX}" != "${N_CACHE_PREFIX}" && ! -e "${N_CACHE_PREFIX}" ]]; then + echo "Folder does not exist: ${N_CACHE_PREFIX}" + echo "- This folder will be created when you do an install." + fi + if [[ -e "${N_PREFIX}" && -e "${N_CACHE_PREFIX}" ]]; then + echo "good" + fi + + if [[ -e "${N_CACHE_PREFIX}" ]]; then + printf "\nChecking permissions for cache folder...\n" + # Using knowledge cache path ends in /n/versions in following check. + if [[ ! -e "${CACHE_DIR}" && (( -e "${N_CACHE_PREFIX}/n" && ! -w "${N_CACHE_PREFIX}/n" ) || ( ! -e "${N_CACHE_PREFIX}/n" && ! -w "${N_CACHE_PREFIX}" )) ]]; then + echo_red "You do not have write permission to create: ${CACHE_DIR}" + show_permission_suggestions + echo "- make a folder you own:" + echo " sudo mkdir -p \"${CACHE_DIR}\"" + echo " sudo chown $(whoami) \"${CACHE_DIR}\"" + elif [[ ! -e "${CACHE_DIR}" ]]; then + echo "Cache folder does not exist: ${CACHE_DIR}" + echo "- This is normal if you have not done an install yet, as cache is only created when needed." + elif [[ ! -w "${CACHE_DIR}" ]]; then + echo_red "You do not have write permission to: ${CACHE_DIR}" + show_permission_suggestions + echo "- change folder ownership to yourself:" + echo " sudo chown -R $(whoami) \"${CACHE_DIR}\"" + else + echo "good" + fi + fi + + if [[ -e "${N_PREFIX}" ]]; then + printf "\nChecking permissions for install folders...\n" + local install_writeable="true" + for subdir in bin lib include share; do + if [[ -e "${N_PREFIX}/${subdir}" && ! -w "${N_PREFIX}/${subdir}" ]]; then + install_writeable="false" + echo_red "You do not have write permission to: ${N_PREFIX}/${subdir}" + break + fi + if [[ ! -e "${N_PREFIX}/${subdir}" && ! -w "${N_PREFIX}" ]]; then + install_writeable="false" + echo_red "You do not have write permission to create: ${N_PREFIX}/${subdir}" + break + fi + done + if [[ "${install_writeable}" = "true" ]]; then + echo "good" + else + show_permission_suggestions + echo "- change folder ownerships to yourself:" + echo " cd \"${N_PREFIX}\"" + echo " sudo mkdir -p bin lib include share" + echo " sudo chown -R $(whoami) bin lib include share" + fi + fi + + printf "\nChecking mirror is reachable...\n" + if is_ok "${N_NODE_MIRROR}/"; then + printf "good\n" + else + echo_red "mirror not reachable" + printf "Showing failing command and output\n" + if command -v curl &> /dev/null; then + ( set -x; do_get --head "${N_NODE_MIRROR}/" ) + else + ( set -x; do_get --spider "${N_NODE_MIRROR}/" ) + printf "\n" + fi + fi +} + +# +# Handle arguments. +# + +# First pass. Process the options so they can come before or after commands, +# particularly for `n lsr --all` and `n install --arch x686` +# which feel pretty natural. + +unprocessed_args=() +positional_arg="false" + +while [[ $# -ne 0 ]]; do + case "$1" in + --all) N_MAX_REMOTE_MATCHES=32000 ;; + -V|--version) display_n_version ;; + -h|--help|help) display_help; exit ;; + -q|--quiet) set_quiet ;; + -d|--download) DOWNLOAD="true" ;; + --offline) OFFLINE="true" ;; + --insecure) set_insecure ;; + -p|--preserve) N_PRESERVE_NPM="true" N_PRESERVE_COREPACK="true" ;; + --no-preserve) N_PRESERVE_NPM="" N_PRESERVE_COREPACK="" ;; + --use-xz) N_USE_XZ="true" ;; + --no-use-xz) N_USE_XZ="false" ;; + --latest) display_remote_versions latest; exit ;; + --stable) display_remote_versions lts; exit ;; # [sic] old terminology + --lts) display_remote_versions lts; exit ;; + -a|--arch) shift; set_arch "$1";; # set arch and continue + exec|run|as|use) + unprocessed_args+=( "$1" ) + positional_arg="true" + ;; + *) + if [[ "${positional_arg}" == "true" ]]; then + unprocessed_args+=( "$@" ) + break + fi + unprocessed_args+=( "$1" ) + ;; + esac + shift +done + +if [[ -z "${N_USE_XZ+defined}" ]]; then + N_USE_XZ="true" # Default to using xz + can_use_xz || N_USE_XZ="false" +fi + +set -- "${unprocessed_args[@]}" + +if test $# -eq 0; then + test -z "$(display_versions_paths)" && err_no_installed_print_help + menu_select_cache_versions +else + while test $# -ne 0; do + case "$1" in + bin|which) display_bin_path_for_version "$2"; exit ;; + run|as|use) shift; run_with_version "$@"; exit ;; + exec) shift; exec_with_version "$@"; exit ;; + doctor) show_diagnostics; exit ;; + rm|-) shift; remove_versions "$@"; exit ;; + prune) prune_cache; exit ;; + latest) install latest; exit ;; + stable) install stable; exit ;; + lts) install lts; exit ;; + ls|list) display_versions_paths; exit ;; + lsr|ls-remote|list-remote) shift; display_remote_versions "$1"; exit ;; + uninstall) uninstall_installed; exit ;; + i|install) shift; install "$1"; exit ;; + N_TEST_DISPLAY_LATEST_RESOLVED_VERSION) shift; get_latest_resolved_version "$1" > /dev/null || exit 2; echo "${g_target_node}"; exit ;; + *) install "$1"; exit ;; + esac + shift + done +fi From 084ecd657865ef024ce423248a4c66c4ee5d2011 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 17:57:56 +0200 Subject: [PATCH 003/146] helpers 2.1: backport most of the bookworm changes --- helpers/helpers.v2.1.d/config | 12 +-- helpers/helpers.v2.1.d/mysql | 3 + helpers/helpers.v2.1.d/php | 163 +++------------------------------ helpers/helpers.v2.1.d/setting | 18 +--- helpers/helpers.v2.1.d/utils | 16 ++-- 5 files changed, 29 insertions(+), 183 deletions(-) diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index d2cc2760e..de35c7744 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -22,7 +22,7 @@ _ynh_app_config_get_one() { if [[ "$bind" == "settings" ]]; then ynh_die --message="File '${short_setting}' can't be stored in settings" fi - old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" + old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" # Get multiline text from settings or from a full file @@ -32,7 +32,7 @@ _ynh_app_config_get_one() { elif [[ "$bind" == *":"* ]]; then ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else - old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" + old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" fi # Get value from a kind of key/value file @@ -47,7 +47,7 @@ _ynh_app_config_get_one() { bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" fi - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key_}" --after="${bind_after}")" fi @@ -73,7 +73,7 @@ _ynh_app_config_apply_one() { if [[ "$bind" == "settings" ]]; then ynh_die --message="File '${short_setting}' can't be stored in settings" fi - local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]]; then ynh_backup_if_checksum_is_different --file="$bind_file" ynh_secure_remove --file="$bind_file" @@ -98,7 +98,7 @@ _ynh_app_config_apply_one() { if [[ "$bind" == *":"* ]]; then ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi - local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" echo "${!short_setting}" >"$bind_file" ynh_store_file_checksum --file="$bind_file" --update_only @@ -113,7 +113,7 @@ _ynh_app_config_apply_one() { bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" fi bind_key_=${bind_key_:-$short_setting} - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}" diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index c11f7989a..6e0ab4a02 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -214,6 +214,9 @@ ynh_mysql_setup_db() { # If $db_pwd is not provided, use new_db_pwd instead for db_pwd db_pwd="${db_pwd:-$new_db_pwd}" + # Dirty patch for super-legacy apps + dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } + ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd } diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 7fbe3f1ba..45e106a1b 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -70,17 +70,14 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vufpdg - local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service [g]=group=) + local legacy_args=vufg + local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [g]=group=) local group local phpversion local usage local footprint - local package - local dedicated_service # Manage arguments with getopts ynh_handle_getopts_args "$@" - package=${package:-} group=${group:-} # The default behaviour is to use the template. @@ -107,8 +104,6 @@ ynh_add_fpm_config() { fi fi - # Do not use a dedicated service by default - dedicated_service=${dedicated_service:-0} # Set the default PHP-FPM version by default if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then @@ -132,45 +127,16 @@ ynh_add_fpm_config() { fi fi - # Legacy args (packager should just list their php dependency as regular apt dependencies... - if [ -n "$package" ]; then - # Install the additionnal packages from the default repository - ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future" - ynh_install_app_dependencies "$package" - fi - - if [ $dedicated_service -eq 1 ]; then - ynh_print_warn --message "Argument --dedicated_service of ynh_add_fpm_config is deprecated and to be removed in the future" - local fpm_service="${app}-phpfpm" - local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" - else - local fpm_service="php${phpversion}-fpm" - local fpm_config_dir="/etc/php/$phpversion/fpm" - fi + local fpm_service="php${phpversion}-fpm" + local fpm_config_dir="/etc/php/$phpversion/fpm" # Create the directory for FPM pools mkdir --parents "$fpm_config_dir/pool.d" ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service" ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion - # Migrate from mutual PHP service to dedicated one. - if [ $dedicated_service -eq 1 ]; then - local old_fpm_config_dir="/etc/php/$phpversion/fpm" - # If a config file exist in the common pool, move it. - if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]; then - ynh_print_info --message="Migrate to a dedicated php-fpm service for $app." - # Create a backup of the old file before migration - ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf" - # Remove the old PHP config file - ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf" - # Reload PHP to release the socket and allow the dedicated service to use it - ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload - fi - fi - if [ $autogenconf == "false" ]; then # Usage 1, use the template in conf/php-fpm.conf local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" @@ -224,56 +190,13 @@ pm.process_idle_timeout = 10s local finalphpconf="$fpm_config_dir/pool.d/$app.conf" ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" - if [ -e "$YNH_APP_BASEDIR/conf/php-fpm.ini" ]; then - ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." - ynh_add_config --template="php-fpm.ini" --destination="$fpm_config_dir/conf.d/20-$app.ini" - fi - - if [ $dedicated_service -eq 1 ]; then - # Create a dedicated php-fpm.conf for the service - local globalphpconf=$fpm_config_dir/php-fpm-$app.conf - - echo "[global] -pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid -error_log = /var/log/php/fpm-php.__APP__.log -syslog.ident = php-fpm-__APP__ -include = __FINALPHPCONF__ -" >$YNH_APP_BASEDIR/conf/php-fpm-$app.conf - - ynh_add_config --template="php-fpm-$app.conf" --destination="$globalphpconf" - - # Create a config for a dedicated PHP-FPM service for the app - echo "[Unit] -Description=PHP __PHPVERSION__ FastCGI Process Manager for __APP__ -After=network.target - -[Service] -Type=notify -PIDFile=/run/php/php__PHPVERSION__-fpm-__APP__.pid -ExecStart=/usr/sbin/php-fpm__PHPVERSION__ --nodaemonize --fpm-config __GLOBALPHPCONF__ -ExecReload=/bin/kill -USR2 \$MAINPID - -[Install] -WantedBy=multi-user.target -" >$YNH_APP_BASEDIR/conf/$fpm_service - - # Create this dedicated PHP-FPM service - ynh_add_systemd_config --service=$fpm_service --template=$fpm_service - # Integrate the service in YunoHost admin panel - yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --description "Php-fpm dedicated to $app" - # Configure log rotate - ynh_use_logrotate --logfile=/var/log/php - # Restart the service, as this service is either stopped or only for this app - ynh_systemd_action --service_name=$fpm_service --action=restart - else - # Validate that the new php conf doesn't break php-fpm entirely - if ! php-fpm${phpversion} --test 2>/dev/null; then - php-fpm${phpversion} --test || true - ynh_secure_remove --file="$finalphpconf" - ynh_die --message="The new configuration broke php-fpm?" - fi - ynh_systemd_action --service_name=$fpm_service --action=reload + # Validate that the new php conf doesn't break php-fpm entirely + if ! php-fpm${phpversion} --test 2>/dev/null; then + php-fpm${phpversion} --test || true + ynh_secure_remove --file="$finalphpconf" + ynh_die --message="The new configuration broke php-fpm?" fi + ynh_systemd_action --service_name=$fpm_service --action=reload } # Remove the dedicated PHP-FPM config @@ -284,8 +207,6 @@ WantedBy=multi-user.target ynh_remove_fpm_config() { local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) - local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) - dedicated_service=${dedicated_service:-0} # Get the version of PHP used by this app local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) @@ -299,69 +220,7 @@ ynh_remove_fpm_config() { fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" - if [ -e $fpm_config_dir/conf.d/20-$app.ini ]; then - ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" - fi - - if [ $dedicated_service -eq 1 ]; then - # Remove the dedicated service PHP-FPM service for the app - ynh_remove_systemd_config --service=$fpm_service - # Remove the global PHP-FPM conf - ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" - # Remove the service from the list of services known by YunoHost - yunohost service remove $fpm_service - elif ynh_package_is_installed --package="php${phpversion}-fpm"; then - ynh_systemd_action --service_name=$fpm_service --action=reload - fi - - # If the PHP version used is not the default version for YunoHost - # The second part with YNH_APP_PURGE is an ugly hack to guess that we're inside the remove script - # (we don't actually care about its value, we just check its not empty hence it exists) - if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - # Remove app dependencies ... but ideally should happen via an explicit call from packager - ynh_remove_app_dependencies - fi -} - -# Install another version of PHP. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_install_php --phpversion=phpversion [--package=packages] -# | arg: -v, --phpversion= - Version of PHP to install. -# | arg: -p, --package= - Additionnal PHP packages to install -# -# Requires YunoHost version 3.8.1 or higher. -ynh_install_php() { - # Declare an array to define the options of this helper. - local legacy_args=vp - local -A args_array=([v]=phpversion= [p]=package=) - local phpversion - local package - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - package=${package:-} - - if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]; then - ynh_die --message="Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION" - fi - - ynh_install_app_dependencies "$package" -} - -# Remove the specific version of PHP used by the app. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_remove_php -# -# Requires YunoHost version 3.8.1 or higher. -ynh_remove_php () { - ynh_remove_app_dependencies + ynh_systemd_action --service_name=$fpm_service --action=reload } # Define the values to configure PHP-FPM diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 82a5d274e..3ddb26851 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -18,11 +18,7 @@ ynh_app_setting_get() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key - else - ynh_app_setting "get" "$app" "$key" - fi + ynh_app_setting "get" "$app" "$key" } # Set an application setting @@ -45,11 +41,7 @@ ynh_app_setting_set() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key -v $value - else - ynh_app_setting "set" "$app" "$key" "$value" - fi + ynh_app_setting "set" "$app" "$key" "$value" } # Delete an application setting @@ -70,11 +62,7 @@ ynh_app_setting_delete() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then - yunohost app setting $app $key -d - else - ynh_app_setting "delete" "$app" "$key" - fi + ynh_app_setting "delete" "$app" "$key" } # Small "hard-coded" interface to avoid calling "yunohost app" directly each diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 631e154e2..9131c2378 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -1074,15 +1074,11 @@ ynh_compare_current_package_version() { _ynh_apply_default_permissions() { local target=$1 - local ynh_requirement=$(ynh_read_manifest --manifest_key="requirements.yunohost" | tr -d '<>= ') - - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then - chmod o-rwx $target - chmod g-w $target - chown -R root:root $target - if ynh_system_user_exists $app; then - chown $app:$app $target - fi + chmod o-rwx $target + chmod g-w $target + chown -R root:root $target + if ynh_system_user_exists $app; then + chown $app:$app $target fi # Crons should be owned by root @@ -1096,7 +1092,7 @@ _ynh_apply_default_permissions() { } int_to_bool() { - sed -e 's/^1$/True/g' -e 's/^0$/False/g' + sed -e 's/^1$/True/g' -e 's/^0$/False/g' -e 's/^true$/True/g' -e 's/^false$/False/g' } toml_to_json() { From b914ad9093ab1303859b1937aefc0f406c354ef5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 18:31:57 +0200 Subject: [PATCH 004/146] helpers 2.1: drop support for 'legacy args' (positionals) in helpers --- helpers/helpers.v2.1.d/apps | 2 - helpers/helpers.v2.1.d/apt | 29 +--- helpers/helpers.v2.1.d/backup | 9 +- helpers/helpers.v2.1.d/config | 2 +- helpers/helpers.v2.1.d/fail2ban | 1 - helpers/helpers.v2.1.d/getopts | 280 +++++++++++++----------------- helpers/helpers.v2.1.d/go | 1 - helpers/helpers.v2.1.d/hardware | 2 - helpers/helpers.v2.1.d/logging | 9 +- helpers/helpers.v2.1.d/logrotate | 1 - helpers/helpers.v2.1.d/mongodb | 12 +- helpers/helpers.v2.1.d/multimedia | 2 - helpers/helpers.v2.1.d/mysql | 9 +- helpers/helpers.v2.1.d/network | 5 - helpers/helpers.v2.1.d/nodejs | 1 - helpers/helpers.v2.1.d/permission | 6 - helpers/helpers.v2.1.d/php | 4 - helpers/helpers.v2.1.d/postgresql | 10 +- helpers/helpers.v2.1.d/ruby | 1 - helpers/helpers.v2.1.d/setting | 5 - helpers/helpers.v2.1.d/string | 5 - helpers/helpers.v2.1.d/systemd | 3 - helpers/helpers.v2.1.d/user | 12 +- helpers/helpers.v2.1.d/utils | 26 +-- 24 files changed, 152 insertions(+), 285 deletions(-) diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps index 81a5717eb..f0c856a23 100644 --- a/helpers/helpers.v2.1.d/apps +++ b/helpers/helpers.v2.1.d/apps @@ -8,7 +8,6 @@ # Requires YunoHost version *.*.* or higher. ynh_install_apps() { # Declare an array to define the options of this helper. - local legacy_args=a local -A args_array=([a]=apps=) local apps # Manage arguments with getopts @@ -127,7 +126,6 @@ ynh_remove_apps() { # If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings. ynh_spawn_app_shell() { # Declare an array to define the options of this helper. - local legacy_args=a local -A args_array=([a]=app=) local app # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index d5a1f4335..51a339544 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -28,7 +28,7 @@ ynh_wait_dpkg_free() { # Check if the name of this file contains only numbers. if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then # If so, that a remaining of dpkg. - ynh_print_err "dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + ynh_print_err --message="dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." set -o xtrace # set -x return 1 fi @@ -43,21 +43,15 @@ ynh_wait_dpkg_free() { # Check either a package is installed or not # -# example: ynh_package_is_installed --package=yunohost && echo "installed" +# example: ynh_package_is_installed foobar && echo "installed" # -# usage: ynh_package_is_installed --package=name -# | arg: -p, --package= - the package name to check +# usage: ynh_package_is_installed name +# | arg: name - the package name to check # | ret: 0 if the package is installed, 1 else. # # Requires YunoHost version 2.2.4 or higher. ynh_package_is_installed() { - # Declare an array to define the options of this helper. - local legacy_args=p - local -A args_array=([p]=package=) - local package - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - + local package=$1 dpkg-query --show --showformat='${Status}' "$package" 2>/dev/null \ | grep --count "ok installed" &>/dev/null } @@ -75,7 +69,6 @@ ynh_package_is_installed() { # Requires YunoHost version 2.2.4 or higher. ynh_package_version() { # Declare an array to define the options of this helper. - local legacy_args=p local -A args_array=([p]=package=) local package # Manage arguments with getopts @@ -310,7 +303,7 @@ ynh_install_app_dependencies() { YNH_INSTALL_APP_DEPENDENCIES_REPLACE="false" else local current_dependencies="" - if ynh_package_is_installed --package="${dep_app}-ynh-deps" + if ynh_package_is_installed "${dep_app}-ynh-deps" then current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) " current_dependencies=${current_dependencies// | /|} @@ -352,7 +345,6 @@ EOF # Requires YunoHost version 3.8.1 or higher. ynh_add_app_dependencies() { # Declare an array to define the options of this helper. - local legacy_args=pr local -A args_array=([p]=package= [r]=replace) local package # Manage arguments with getopts @@ -375,7 +367,7 @@ ynh_remove_app_dependencies() { local dep_app=${app//_/-} # Replace all '_' by '-' local current_dependencies="" - if ynh_package_is_installed --package="${dep_app}-ynh-deps"; then + if ynh_package_is_installed "${dep_app}-ynh-deps"; then current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) " current_dependencies=${current_dependencies// | /|} fi @@ -409,7 +401,6 @@ ynh_remove_app_dependencies() { # Requires YunoHost version 3.8.1 or higher. ynh_install_extra_app_dependencies() { # Declare an array to define the options of this helper. - local legacy_args=rpkn local -A args_array=([r]=repo= [p]=package= [k]=key= [n]=name=) local repo local package @@ -454,7 +445,6 @@ ynh_install_extra_app_dependencies() { # Requires YunoHost version 3.8.1 or higher. ynh_install_extra_repo() { # Declare an array to define the options of this helper. - local legacy_args=rkpna local -A args_array=([r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append) local repo local key @@ -523,7 +513,6 @@ ynh_install_extra_repo() { # Requires YunoHost version 3.8.1 or higher. ynh_remove_extra_repo() { # Declare an array to define the options of this helper. - local legacy_args=n local -A args_array=([n]=name=) local name # Manage arguments with getopts @@ -532,7 +521,7 @@ ynh_remove_extra_repo() { ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list" # Sury pinning is managed by the regenconf in the core... - [[ "$name" == "extra_php_version" ]] || ynh_secure_remove "/etc/apt/preferences.d/$name" + [[ "$name" == "extra_php_version" ]] || ynh_secure_remove --file="/etc/apt/preferences.d/$name" if [ -e /etc/apt/trusted.gpg.d/$name.gpg ]; then ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg" fi @@ -564,7 +553,6 @@ ynh_remove_extra_repo() { # Requires YunoHost version 3.8.1 or higher. ynh_add_repo() { # Declare an array to define the options of this helper. - local legacy_args=uscna local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append) local uri local suite @@ -604,7 +592,6 @@ ynh_add_repo() { # Requires YunoHost version 3.8.1 or higher. ynh_pin_repo() { # Declare an array to define the options of this helper. - local legacy_args=pirna local -A args_array=([p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append) local package local pin diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index a596ac9e0..f08495923 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -65,7 +65,6 @@ ynh_backup() { # TODO find a way to avoid injection by file strange naming ! # Declare an array to define the options of this helper. - local legacy_args=sdbm local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory) local src_path local dest_path @@ -237,7 +236,6 @@ with open(sys.argv[1], 'r') as backup_file: # Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory ynh_restore_file() { # Declare an array to define the options of this helper. - local legacy_args=odm local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory) local origin_path local dest_path @@ -305,7 +303,6 @@ ynh_restore_file() { # Requires YunoHost version 2.6.4 or higher. ynh_store_file_checksum() { # Declare an array to define the options of this helper. - local legacy_args=f local -A args_array=([f]=file= [u]=update_only) local file local update_only @@ -356,7 +353,6 @@ ynh_store_file_checksum() { # Requires YunoHost version 2.6.4 or higher. ynh_backup_if_checksum_is_different() { # Declare an array to define the options of this helper. - local legacy_args=f local -A args_array=([f]=file=) local file # Manage arguments with getopts @@ -372,13 +368,13 @@ ynh_backup_if_checksum_is_different() { backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')" mkdir --parents "$(dirname "$backup_file_checksum")" cp --archive "$file" "$backup_file_checksum" # Backup the current file - ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + ynh_print_warn --message="File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" echo "$backup_file_checksum" # Return the name of the backup file if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then local file_path_base64=$(echo "$file" | base64 -w0) if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64} then - ynh_print_warn "Diff with the original file:" + ynh_print_warn --message="Diff with the original file:" diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true fi fi @@ -396,7 +392,6 @@ ynh_backup_if_checksum_is_different() { # Requires YunoHost version 3.3.1 or higher. ynh_delete_file_checksum() { # Declare an array to define the options of this helper. - local legacy_args=f local -A args_array=([f]=file=) local file # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index de35c7744..6fdc74cd9 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -320,7 +320,7 @@ ynh_app_action_run() { #ynh_return "result:" #ynh_return "$(echo "${result}" | sed 's/^/ /g')" else - ynh_die "No handler defined in app's script for action $1. If you are the maintainer of this app, you should define '$runner'" + ynh_die --message="No handler defined in app's script for action $1. If you are the maintainer of this app, you should define '$runner'" fi } diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index 613dcc490..9343d2bea 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -59,7 +59,6 @@ # Requires YunoHost version 4.1.0 or higher. ynh_add_fail2ban_config() { # Declare an array to define the options of this helper. - local legacy_args=lrmptv local -A args_array=([l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template) local logpath local failregex diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index f9ef5dc0b..1e32bc982 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -48,168 +48,138 @@ ynh_handle_getopts_args() { # Manage arguments only if there's some provided set +o xtrace # set +x - if [ $# -ne 0 ]; then - # Store arguments in an array to keep each argument separated - local arguments=("$@") + if [ $# -eq 0 ]; then + set -o xtrace # set -x + return + fi - # For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u) - # And built parameters string for getopts - # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value) - local getopts_parameters="" - local option_flag="" - for option_flag in "${!args_array[@]}"; do - # Concatenate each option_flags of the array to build the string of arguments for getopts - # Will looks like 'abcd' for -a -b -c -d - # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob) - # Check the last character of the value associate to the option_flag - if [ "${args_array[$option_flag]: -1}" = "=" ]; then - # For an option with additionnal values, add a ':' after the letter for getopts. - getopts_parameters="${getopts_parameters}${option_flag}:" - else - getopts_parameters="${getopts_parameters}${option_flag}" - fi - # Check each argument given to the function - local arg="" - # ${#arguments[@]} is the size of the array - for arg in $(seq 0 $((${#arguments[@]} - 1))); do - # Escape options' values starting with -. Otherwise the - will be considered as another option. - arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}" - # And replace long option (value of the option_flag) by the short option, the option_flag itself - # (e.g. for [u]=user, --user will be -u) - # Replace long option with = (match the beginning of the argument) - arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")" - # And long option without = (match the whole line) - arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")" - done + # Store arguments in an array to keep each argument separated + local arguments=("$@") + + # For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u) + # And built parameters string for getopts + # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value) + local getopts_parameters="" + local option_flag="" + for option_flag in "${!args_array[@]}"; do + # Concatenate each option_flags of the array to build the string of arguments for getopts + # Will looks like 'abcd' for -a -b -c -d + # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob) + # Check the last character of the value associate to the option_flag + if [ "${args_array[$option_flag]: -1}" = "=" ]; then + # For an option with additionnal values, add a ':' after the letter for getopts. + getopts_parameters="${getopts_parameters}${option_flag}:" + else + getopts_parameters="${getopts_parameters}${option_flag}" + fi + # Check each argument given to the function + local arg="" + # ${#arguments[@]} is the size of the array + for arg in $(seq 0 $((${#arguments[@]} - 1))); do + # Escape options' values starting with -. Otherwise the - will be considered as another option. + arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}" + # And replace long option (value of the option_flag) by the short option, the option_flag itself + # (e.g. for [u]=user, --user will be -u) + # Replace long option with = (match the beginning of the argument) + arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")" + # And long option without = (match the whole line) + arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")" done + done - # Read and parse all the arguments - # Use a function here, to use standart arguments $@ and be able to use shift. - parse_arg() { - # Read all arguments, until no arguments are left - while [ $# -ne 0 ]; do - # Initialize the index of getopts - OPTIND=1 - # Parse with getopts only if the argument begin by -, that means the argument is an option - # getopts will fill $parameter with the letter of the option it has read. - local parameter="" - getopts ":$getopts_parameters" parameter || true + # Read and parse all the arguments + # Use a function here, to use standart arguments $@ and be able to use shift. + parse_arg() { + # Read all arguments, until no arguments are left + while [ $# -ne 0 ]; do + # Initialize the index of getopts + OPTIND=1 + # Parse with getopts only if the argument begin by -, that means the argument is an option + # getopts will fill $parameter with the letter of the option it has read. + local parameter="" + getopts ":$getopts_parameters" parameter || true - if [ "$parameter" = "?" ]; then - ynh_die --message="Invalid argument: -${OPTARG:-}" - elif [ "$parameter" = ":" ]; then - ynh_die --message="-$OPTARG parameter requires an argument." - else - local shift_value=1 - # Use the long option, corresponding to the short option read by getopts, as a variable - # (e.g. for [u]=user, 'user' will be used as a variable) - # Also, remove '=' at the end of the long option - # The variable name will be stored in 'option_var' - local option_var="${args_array[$parameter]%=}" - # If this option doesn't take values - # if there's a '=' at the end of the long option name, this option takes values - if [ "${args_array[$parameter]: -1}" != "=" ]; then - # 'eval ${option_var}' will use the content of 'option_var' - eval ${option_var}=1 - else - # Read all other arguments to find multiple value for this option. - # Load args in a array - local all_args=("$@") - - # If the first argument is longer than 2 characters, - # There's a value attached to the option, in the same array cell - if [ ${#all_args[0]} -gt 2 ]; then - # Remove the option and the space, so keep only the value itself. - all_args[0]="${all_args[0]#-${parameter} }" - - # At this point, if all_args[0] start with "-", then the argument is not well formed - if [ "${all_args[0]:0:1}" == "-" ]; then - ynh_die --message="Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?" - fi - # Reduce the value of shift, because the option has been removed manually - shift_value=$((shift_value - 1)) - fi - - # Declare the content of option_var as a variable. - eval ${option_var}="" - # Then read the array value per value - local i - for i in $(seq 0 $((${#all_args[@]} - 1))); do - # If this argument is an option, end here. - if [ "${all_args[$i]:0:1}" == "-" ]; then - # Ignore the first value of the array, which is the option itself - if [ "$i" -ne 0 ]; then - break - fi - else - # Ignore empty parameters - if [ -n "${all_args[$i]}" ]; then - # Else, add this value to this option - # Each value will be separated by ';' - if [ -n "${!option_var}" ]; then - # If there's already another value for this option, add a ; before adding the new value - eval ${option_var}+="\;" - fi - - # Remove the \ that escape - at beginning of values. - all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}" - - # For the record. - # We're using eval here to get the content of the variable stored itself as simple text in $option_var... - # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var} - # But... ${!option_var} can't be used as left part of an assignation. - # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself. - # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one! - - eval ${option_var}+='"${all_args[$i]}"' - fi - shift_value=$((shift_value + 1)) - fi - done - fi - fi - - # Shift the parameter and its argument(s) - shift $shift_value - done - } - - # LEGACY MODE - # Check if there's getopts arguments - if [ "${arguments[0]:0:1}" != "-" ]; then - # If not, enter in legacy mode and manage the arguments as positionnal ones.. - # Dot not echo, to prevent to go through a helper output. But print only in the log. - set -x - echo "! Helper used in legacy mode !" >/dev/null - set +x - local i - for i in $(seq 0 $((${#arguments[@]} - 1))); do - # Try to use legacy_args as a list of option_flag of the array args_array - # Otherwise, fallback to getopts_parameters to get the option_flag. But an associative arrays isn't always sorted in the correct order... - # Remove all ':' in getopts_parameters - getopts_parameters=${legacy_args:-${getopts_parameters//:/}} - # Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument. - option_flag=${getopts_parameters:$i:1} - if [ -z "$option_flag" ]; then - ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored." - continue - fi - # Use the long option, corresponding to the option_flag, as a variable + if [ "$parameter" = "?" ]; then + ynh_die --message="Invalid argument: -${OPTARG:-}" + elif [ "$parameter" = ":" ]; then + ynh_die --message="-$OPTARG parameter requires an argument." + else + local shift_value=1 + # Use the long option, corresponding to the short option read by getopts, as a variable # (e.g. for [u]=user, 'user' will be used as a variable) # Also, remove '=' at the end of the long option # The variable name will be stored in 'option_var' - local option_var="${args_array[$option_flag]%=}" + local option_var="${args_array[$parameter]%=}" + # If this option doesn't take values + # if there's a '=' at the end of the long option name, this option takes values + if [ "${args_array[$parameter]: -1}" != "=" ]; then + # 'eval ${option_var}' will use the content of 'option_var' + eval ${option_var}=1 + else + # Read all other arguments to find multiple value for this option. + # Load args in a array + local all_args=("$@") - # Store each value given as argument in the corresponding variable - # The values will be stored in the same order than $args_array - eval ${option_var}+='"${arguments[$i]}"' - done - unset legacy_args - else - # END LEGACY MODE - # Call parse_arg and pass the modified list of args as an array of arguments. - parse_arg "${arguments[@]}" - fi - fi + # If the first argument is longer than 2 characters, + # There's a value attached to the option, in the same array cell + if [ ${#all_args[0]} -gt 2 ]; then + # Remove the option and the space, so keep only the value itself. + all_args[0]="${all_args[0]#-${parameter} }" + + # At this point, if all_args[0] start with "-", then the argument is not well formed + if [ "${all_args[0]:0:1}" == "-" ]; then + ynh_die --message="Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?" + fi + # Reduce the value of shift, because the option has been removed manually + shift_value=$((shift_value - 1)) + fi + + # Declare the content of option_var as a variable. + eval ${option_var}="" + # Then read the array value per value + local i + for i in $(seq 0 $((${#all_args[@]} - 1))); do + # If this argument is an option, end here. + if [ "${all_args[$i]:0:1}" == "-" ]; then + # Ignore the first value of the array, which is the option itself + if [ "$i" -ne 0 ]; then + break + fi + else + # Ignore empty parameters + if [ -n "${all_args[$i]}" ]; then + # Else, add this value to this option + # Each value will be separated by ';' + if [ -n "${!option_var}" ]; then + # If there's already another value for this option, add a ; before adding the new value + eval ${option_var}+="\;" + fi + + # Remove the \ that escape - at beginning of values. + all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}" + + # For the record. + # We're using eval here to get the content of the variable stored itself as simple text in $option_var... + # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var} + # But... ${!option_var} can't be used as left part of an assignation. + # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself. + # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one! + + eval ${option_var}+='"${all_args[$i]}"' + fi + shift_value=$((shift_value + 1)) + fi + done + fi + fi + + # Shift the parameter and its argument(s) + shift $shift_value + done + } + + # Call parse_arg and pass the modified list of args as an array of arguments. + parse_arg "${arguments[@]}" + set -o xtrace # set -x } diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 0e18301f7..c4b0643c8 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -95,7 +95,6 @@ ynh_use_go () { # Requires YunoHost version 3.2.2 or higher. ynh_install_go () { # Declare an array to define the options of this helper. - local legacy_args=v local -A args_array=( [v]=go_version= ) local go_version # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/hardware b/helpers/helpers.v2.1.d/hardware index 091f023f6..aed1a18ba 100644 --- a/helpers/helpers.v2.1.d/hardware +++ b/helpers/helpers.v2.1.d/hardware @@ -14,7 +14,6 @@ # Requires YunoHost version 3.8.1 or higher. ynh_get_ram() { # Declare an array to define the options of this helper. - local legacy_args=ftso local -A args_array=([f]=free [t]=total [s]=ignore_swap [o]=only_swap) local free local total @@ -78,7 +77,6 @@ ynh_get_ram() { # Requires YunoHost version 3.8.1 or higher. ynh_require_ram() { # Declare an array to define the options of this helper. - local legacy_args=rftso local -A args_array=([r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap) local required local free diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index accb8f9b0..d761a20e5 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -9,7 +9,6 @@ # Requires YunoHost version 2.4.0 or higher. ynh_die() { # Declare an array to define the options of this helper. - local legacy_args=mc local -A args_array=([m]=message= [c]=ret_code=) local message local ret_code @@ -29,7 +28,6 @@ ynh_die() { # Requires YunoHost version 3.2.0 or higher. ynh_print_info() { # Declare an array to define the options of this helper. - local legacy_args=m local -A args_array=([m]=message=) local message # Manage arguments with getopts @@ -55,7 +53,6 @@ ynh_print_log() { # Requires YunoHost version 3.2.0 or higher. ynh_print_warn() { # Declare an array to define the options of this helper. - local legacy_args=m local -A args_array=([m]=message=) local message # Manage arguments with getopts @@ -72,7 +69,6 @@ ynh_print_warn() { # Requires YunoHost version 3.2.0 or higher. ynh_print_err() { # Declare an array to define the options of this helper. - local legacy_args=m local -A args_array=([m]=message=) local message # Manage arguments with getopts @@ -201,7 +197,7 @@ ynh_exec_and_print_stderr_only_if_error() { "$@" 2> "$logfile" || rc="$?" if (( rc != 0 )); then ynh_exec_warn cat "$logfile" - ynh_secure_remove "$logfile" + ynh_secure_remove --file="$logfile" return "$rc" fi } @@ -257,7 +253,6 @@ base_time=$(date +%s) ynh_script_progression() { set +o xtrace # set +x # Declare an array to define the options of this helper. - local legacy_args=mwtl local -A args_array=([m]=message= [w]=weight= [t]=time [l]=last) local message local weight @@ -332,7 +327,7 @@ ynh_script_progression() { print_exec_time=" [$(bc <<< "scale=1; $exec_time / 60" ) minutes]" fi - ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" + echo "[$progression_bar] > ${message}${print_exec_time}" >&$YNH_STDINFO set -o xtrace # set -x } diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index efc1137c1..4f5cb69e9 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -27,7 +27,6 @@ ynh_use_logrotate() { set -- "${all_args[@]}" # Argument parsing - local legacy_args=lu local -A args_array=([l]=logfile= [u]=specific_user=) local logfile local specific_user diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index d40d11bfe..c8b016e1b 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -18,7 +18,6 @@ # ynh_mongo_exec() { # Declare an array to define the options of this helper. - local legacy_args=upadhPce local -A args_array=( [u]=user= [p]=password= [a]=authenticationdatabase= [d]=database= [h]=host= [P]=port= [c]=command= [e]=eval ) local user local password @@ -115,7 +114,6 @@ EOF # ynh_mongo_drop_db() { # Declare an array to define the options of this helper. - local legacy_args=d local -A args_array=( [d]=database= ) local database # Manage arguments with getopts @@ -135,7 +133,6 @@ ynh_mongo_drop_db() { # ynh_mongo_dump_db() { # Declare an array to define the options of this helper. - local legacy_args=d local -A args_array=( [d]=database= ) local database # Manage arguments with getopts @@ -156,7 +153,6 @@ ynh_mongo_dump_db() { # ynh_mongo_create_user() { # Declare an array to define the options of this helper. - local legacy_args=unp local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) local db_user local db_name @@ -180,7 +176,6 @@ ynh_mongo_create_user() { # ynh_mongo_database_exists() { # Declare an array to define the options of this helper. - local legacy_args=d local -A args_array=([d]=database=) local database # Manage arguments with getopts @@ -204,7 +199,6 @@ ynh_mongo_database_exists() { # ynh_mongo_restore_db() { # Declare an array to define the options of this helper. - local legacy_args=d local -A args_array=( [d]=database= ) local database # Manage arguments with getopts @@ -224,7 +218,6 @@ ynh_mongo_restore_db() { # ynh_mongo_drop_user() { # Declare an array to define the options of this helper. - local legacy_args=un local -A args_array=( [u]=db_user= [n]=db_name= ) local db_user local db_name @@ -247,7 +240,6 @@ ynh_mongo_drop_user() { # ynh_mongo_setup_db() { # Declare an array to define the options of this helper. - local legacy_args=unp local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) local db_user local db_name @@ -275,7 +267,6 @@ ynh_mongo_setup_db() { # ynh_mongo_remove_db() { # Declare an array to define the options of this helper. - local legacy_args=un local -A args_array=( [u]=db_user= [n]=db_name= ) local db_user local db_name @@ -300,7 +291,6 @@ ynh_mongo_remove_db() { # ynh_install_mongo() { # Declare an array to define the options of this helper. - local legacy_args=m local -A args_array=([m]=mongo_version=) local mongo_version # Manage arguments with getopts @@ -343,7 +333,7 @@ ynh_install_mongo() { # ynh_remove_mongo() { # Only remove the mongodb service if it is not installed. - if ! ynh_package_is_installed --package="mongodb*" + if ! ynh_package_is_installed "mongodb*" then ynh_print_info --message="Removing MongoDB service..." mongodb_servicename=mongod diff --git a/helpers/helpers.v2.1.d/multimedia b/helpers/helpers.v2.1.d/multimedia index c860ae49f..b1d8a526e 100644 --- a/helpers/helpers.v2.1.d/multimedia +++ b/helpers/helpers.v2.1.d/multimedia @@ -64,7 +64,6 @@ ynh_multimedia_build_main_dir() { ynh_multimedia_addfolder() { # Declare an array to define the options of this helper. - local legacy_args=sd local -A args_array=([s]=source_dir= [d]=dest_dir=) local source_dir local dest_dir @@ -92,7 +91,6 @@ ynh_multimedia_addfolder() { # Requires YunoHost version 4.2 or higher. ynh_multimedia_addaccess() { # Declare an array to define the options of this helper. - local legacy_args=u declare -Ar args_array=([u]=user_name=) local user_name # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index 6e0ab4a02..7b7e0fd31 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -14,7 +14,6 @@ # Requires YunoHost version 2.2.4 or higher. ynh_mysql_connect_as() { # Declare an array to define the options of this helper. - local legacy_args=upd local -A args_array=([u]=user= [p]=password= [d]=database=) local user local password @@ -35,7 +34,6 @@ ynh_mysql_connect_as() { # Requires YunoHost version 2.2.4 or higher. ynh_mysql_execute_as_root() { # Declare an array to define the options of this helper. - local legacy_args=sd local -A args_array=([s]=sql= [d]=database=) local sql local database @@ -59,7 +57,6 @@ ynh_mysql_execute_as_root() { # Requires YunoHost version 2.2.4 or higher. ynh_mysql_execute_file_as_root() { # Declare an array to define the options of this helper. - local legacy_args=fd local -A args_array=([f]=file= [d]=database=) local file local database @@ -127,7 +124,6 @@ ynh_mysql_drop_db() { # Requires YunoHost version 2.2.4 or higher. ynh_mysql_dump_db() { # Declare an array to define the options of this helper. - local legacy_args=d local -A args_array=([d]=database=) local database # Manage arguments with getopts @@ -161,7 +157,6 @@ ynh_mysql_create_user() { # Requires YunoHost version 2.2.4 or higher. ynh_mysql_user_exists() { # Declare an array to define the options of this helper. - local legacy_args=u local -A args_array=([u]=user=) local user # Manage arguments with getopts @@ -201,7 +196,6 @@ ynh_mysql_drop_user() { # Requires YunoHost version 2.6.4 or higher. ynh_mysql_setup_db() { # Declare an array to define the options of this helper. - local legacy_args=unp local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) local db_user local db_name @@ -215,7 +209,7 @@ ynh_mysql_setup_db() { db_pwd="${db_pwd:-$new_db_pwd}" # Dirty patch for super-legacy apps - dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } + dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn --message="Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd @@ -232,7 +226,6 @@ ynh_mysql_setup_db() { # Requires YunoHost version 2.6.4 or higher. ynh_mysql_remove_db() { # Declare an array to define the options of this helper. - local legacy_args=un local -Ar args_array=([u]=db_user= [n]=db_name=) local db_user local db_name diff --git a/helpers/helpers.v2.1.d/network b/helpers/helpers.v2.1.d/network index bed9dd402..c407af512 100644 --- a/helpers/helpers.v2.1.d/network +++ b/helpers/helpers.v2.1.d/network @@ -13,7 +13,6 @@ # Requires YunoHost version 2.6.4 or higher. ynh_find_port() { # Declare an array to define the options of this helper. - local legacy_args=p local -A args_array=([p]=port=) local port # Manage arguments with getopts @@ -39,7 +38,6 @@ ynh_find_port() { # Requires YunoHost version 3.8.0 or higher. ynh_port_available() { # Declare an array to define the options of this helper. - local legacy_args=p local -A args_array=([p]=port=) local port # Manage arguments with getopts @@ -70,7 +68,6 @@ ynh_validate_ip() { # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 # Declare an array to define the options of this helper. - local legacy_args=fi local -A args_array=([f]=family= [i]=ip_address=) local family local ip_address @@ -102,7 +99,6 @@ EOF # Requires YunoHost version 2.2.4 or higher. ynh_validate_ip4() { # Declare an array to define the options of this helper. - local legacy_args=i local -A args_array=([i]=ip_address=) local ip_address # Manage arguments with getopts @@ -122,7 +118,6 @@ ynh_validate_ip4() { # Requires YunoHost version 2.2.4 or higher. ynh_validate_ip6() { # Declare an array to define the options of this helper. - local legacy_args=i local -A args_array=([i]=ip_address=) local ip_address # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index 779f077d0..4bf9a0b71 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -95,7 +95,6 @@ ynh_install_nodejs() { # Use n, https://github.com/tj/n to manage the nodejs versions # Declare an array to define the options of this helper. - local legacy_args=n local -A args_array=([n]=nodejs_version=) local nodejs_version # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/permission b/helpers/helpers.v2.1.d/permission index d3eb71c22..bf2ca86ed 100644 --- a/helpers/helpers.v2.1.d/permission +++ b/helpers/helpers.v2.1.d/permission @@ -67,7 +67,6 @@ # Requires YunoHost version 3.7.0 or higher. ynh_permission_create() { # Declare an array to define the options of this helper. - local legacy_args=puAhaltP local -A args_array=([p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected=) local permission local url @@ -155,7 +154,6 @@ ynh_permission_create() { # Requires YunoHost version 3.7.0 or higher. ynh_permission_delete() { # Declare an array to define the options of this helper. - local legacy_args=p local -A args_array=([p]=permission=) local permission ynh_handle_getopts_args "$@" @@ -174,7 +172,6 @@ ynh_permission_delete() { # Requires YunoHost version 3.7.0 or higher. ynh_permission_exists() { # Declare an array to define the options of this helper. - local legacy_args=p local -A args_array=([p]=permission=) local permission ynh_handle_getopts_args "$@" @@ -199,7 +196,6 @@ ynh_permission_exists() { # Requires YunoHost version 3.7.0 or higher. ynh_permission_url() { # Declare an array to define the options of this helper. - local legacy_args=puarhc local -A args_array=([p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls) local permission local url @@ -269,7 +265,6 @@ ynh_permission_url() { # Requires YunoHost version 3.7.0 or higher. ynh_permission_update() { # Declare an array to define the options of this helper. - local legacy_args=parltP local -A args_array=([p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected=) local permission local add @@ -337,7 +332,6 @@ ynh_permission_update() { # # Requires YunoHost version 3.7.1 or higher. ynh_permission_has_user() { - local legacy_args=pu # Declare an array to define the options of this helper. local -A args_array=([p]=permission= [u]=user=) local permission diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 45e106a1b..52100560d 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -70,7 +70,6 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vufg local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [g]=group=) local group local phpversion @@ -243,7 +242,6 @@ ynh_remove_fpm_config() { # # | arg: -p, --print - Print the result (intended for debug purpose only when packaging the app) ynh_get_scalable_phpfpm() { - local legacy_args=ufp # Declare an array to define the options of this helper. local -A args_array=([u]=usage= [f]=footprint= [p]=print) local usage @@ -376,7 +374,6 @@ YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} ynh_composer_exec() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vwc declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=) local phpversion local workdir @@ -408,7 +405,6 @@ ynh_composer_exec() { ynh_install_composer() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vwac declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) local phpversion local workdir diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 35b95cd5f..09c906cb6 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -17,7 +17,6 @@ PSQL_VERSION=13 # Requires YunoHost version 3.5.0 or higher. ynh_psql_connect_as() { # Declare an array to define the options of this helper. - local legacy_args=upd local -A args_array=([u]=user= [p]=password= [d]=database=) local user local password @@ -38,7 +37,6 @@ ynh_psql_connect_as() { # Requires YunoHost version 3.5.0 or higher. ynh_psql_execute_as_root() { # Declare an array to define the options of this helper. - local legacy_args=sd local -A args_array=([s]=sql= [d]=database=) local sql local database @@ -63,7 +61,6 @@ ynh_psql_execute_as_root() { # Requires YunoHost version 3.5.0 or higher. ynh_psql_execute_file_as_root() { # Declare an array to define the options of this helper. - local legacy_args=fd local -A args_array=([f]=file= [d]=database=) local file local database @@ -134,7 +131,6 @@ ynh_psql_drop_db() { # Requires YunoHost version 3.5.0 or higher. ynh_psql_dump_db() { # Declare an array to define the options of this helper. - local legacy_args=d local -A args_array=([d]=database=) local database # Manage arguments with getopts @@ -169,7 +165,6 @@ ynh_psql_create_user() { # Requires YunoHost version 3.5.0 or higher. ynh_psql_user_exists() { # Declare an array to define the options of this helper. - local legacy_args=u local -A args_array=([u]=user=) local user # Manage arguments with getopts @@ -191,7 +186,6 @@ ynh_psql_user_exists() { # Requires YunoHost version 3.5.0 or higher. ynh_psql_database_exists() { # Declare an array to define the options of this helper. - local legacy_args=d local -A args_array=([d]=database=) local database # Manage arguments with getopts @@ -201,7 +195,7 @@ ynh_psql_database_exists() { # though it could exists. if ! command -v psql then - ynh_print_err -m "PostgreSQL is not installed, impossible to check for db existence." + ynh_print_err --message="PostgreSQL is not installed, impossible to check for db existence." return 1 elif ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then return 1 @@ -237,7 +231,6 @@ ynh_psql_drop_user() { # Requires YunoHost version 2.7.13 or higher. ynh_psql_setup_db() { # Declare an array to define the options of this helper. - local legacy_args=unp local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) local db_user local db_name @@ -270,7 +263,6 @@ ynh_psql_setup_db() { # Requires YunoHost version 2.7.13 or higher. ynh_psql_remove_db() { # Declare an array to define the options of this helper. - local legacy_args=un local -A args_array=([u]=db_user= [n]=db_name=) local db_user local db_name diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 82a946935..10e450ed9 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -97,7 +97,6 @@ ynh_use_ruby () { # Requires YunoHost version 3.2.2 or higher. ynh_install_ruby () { # Declare an array to define the options of this helper. - local legacy_args=v local -A args_array=( [v]=ruby_version= ) local ruby_version # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 3ddb26851..bd9aa2ab4 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -10,7 +10,6 @@ ynh_app_setting_get() { local _globalapp=${app-:} # Declare an array to define the options of this helper. - local legacy_args=ak local -A args_array=([a]=app= [k]=key=) local app local key @@ -32,7 +31,6 @@ ynh_app_setting_get() { ynh_app_setting_set() { local _globalapp=${app-:} # Declare an array to define the options of this helper. - local legacy_args=akv local -A args_array=([a]=app= [k]=key= [v]=value=) local app local key @@ -54,7 +52,6 @@ ynh_app_setting_set() { ynh_app_setting_delete() { local _globalapp=${app-:} # Declare an array to define the options of this helper. - local legacy_args=ak local -A args_array=([a]=app= [k]=key=) local app local key @@ -112,7 +109,6 @@ EOF # Requires YunoHost version 2.6.4 or higher. ynh_webpath_available() { # Declare an array to define the options of this helper. - local legacy_args=dp local -A args_array=([d]=domain= [p]=path_url=) local domain local path_url @@ -136,7 +132,6 @@ ynh_webpath_available() { # Requires YunoHost version 2.6.4 or higher. ynh_webpath_register() { # Declare an array to define the options of this helper. - local legacy_args=adp local -A args_array=([a]=app= [d]=domain= [p]=path_url=) local app local domain diff --git a/helpers/helpers.v2.1.d/string b/helpers/helpers.v2.1.d/string index b674d9a4a..f022deab2 100644 --- a/helpers/helpers.v2.1.d/string +++ b/helpers/helpers.v2.1.d/string @@ -12,7 +12,6 @@ # Requires YunoHost version 2.2.4 or higher. ynh_string_random() { # Declare an array to define the options of this helper. - local legacy_args=lf local -A args_array=([l]=length= [f]=filter=) local length local filter @@ -39,7 +38,6 @@ ynh_string_random() { # Requires YunoHost version 2.6.4 or higher. ynh_replace_string() { # Declare an array to define the options of this helper. - local legacy_args=mrf local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) local match_string local replace_string @@ -70,7 +68,6 @@ ynh_replace_string() { # Requires YunoHost version 2.7.7 or higher. ynh_replace_special_string() { # Declare an array to define the options of this helper. - local legacy_args=mrf local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) local match_string local replace_string @@ -104,7 +101,6 @@ ynh_replace_special_string() { # Requires YunoHost version 2.2.4 or higher. ynh_sanitize_dbid() { # Declare an array to define the options of this helper. - local legacy_args=n local -A args_array=([n]=db_name=) local db_name # Manage arguments with getopts @@ -134,7 +130,6 @@ ynh_sanitize_dbid() { # Requires YunoHost version 2.6.4 or higher. ynh_normalize_url_path() { # Declare an array to define the options of this helper. - local legacy_args=p local -A args_array=([p]=path_url=) local path_url # Manage arguments with getopts diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 765c575ef..57d6b3802 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -14,7 +14,6 @@ # Requires YunoHost version 4.1.0 or higher. ynh_add_systemd_config() { # Declare an array to define the options of this helper. - local legacy_args=stv local -A args_array=([s]=service= [t]=template=) local service local template @@ -37,7 +36,6 @@ ynh_add_systemd_config() { # Requires YunoHost version 2.7.2 or higher. ynh_remove_systemd_config() { # Declare an array to define the options of this helper. - local legacy_args=s local -A args_array=([s]=service=) local service # Manage arguments with getopts @@ -66,7 +64,6 @@ ynh_remove_systemd_config() { # Requires YunoHost version 3.5.0 or higher. ynh_systemd_action() { # Declare an array to define the options of this helper. - local legacy_args=nalpte local -A args_array=([n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=) local service_name local action diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index e608a3308..14a8e32b5 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -11,7 +11,6 @@ # Requires YunoHost version 2.2.4 or higher. ynh_user_exists() { # Declare an array to define the options of this helper. - local legacy_args=u local -A args_array=([u]=username=) local username # Manage arguments with getopts @@ -32,7 +31,6 @@ ynh_user_exists() { # Requires YunoHost version 2.2.4 or higher. ynh_user_get_info() { # Declare an array to define the options of this helper. - local legacy_args=uk local -A args_array=([u]=username= [k]=key=) local username local key @@ -65,7 +63,6 @@ ynh_user_list() { # Requires YunoHost version 2.2.4 or higher. ynh_system_user_exists() { # Declare an array to define the options of this helper. - local legacy_args=u local -A args_array=([u]=username=) local username # Manage arguments with getopts @@ -85,7 +82,6 @@ ynh_system_user_exists() { # Requires YunoHost version 3.5.0.2 or higher. ynh_system_group_exists() { # Declare an array to define the options of this helper. - local legacy_args=g local -A args_array=([g]=group=) local group # Manage arguments with getopts @@ -116,7 +112,6 @@ ynh_system_group_exists() { # Requires YunoHost version 2.6.4 or higher. ynh_system_user_create() { # Declare an array to define the options of this helper. - local legacy_args=uhs local -A args_array=([u]=username= [h]=home_dir= [s]=use_shell [g]=groups=) local username local home_dir @@ -129,7 +124,7 @@ ynh_system_user_create() { home_dir="${home_dir:-}" groups="${groups:-}" - if ! ynh_system_user_exists "$username"; then # Check if the user exists on the system + if ! ynh_system_user_exists --username="$username"; then # Check if the user exists on the system # If the user doesn't exist if [ -n "$home_dir" ]; then # If a home dir is mentioned local user_home_dir="--home-dir $home_dir" @@ -160,21 +155,20 @@ ynh_system_user_create() { # Requires YunoHost version 2.6.4 or higher. ynh_system_user_delete() { # Declare an array to define the options of this helper. - local legacy_args=u local -A args_array=([u]=username=) local username # Manage arguments with getopts ynh_handle_getopts_args "$@" # Check if the user exists on the system - if ynh_system_user_exists "$username"; then + if ynh_system_user_exists --username="$username"; then deluser $username else ynh_print_warn --message="The user $username was not found" fi # Check if the group exists on the system - if ynh_system_group_exists "$username"; then + if ynh_system_group_exists --group="$username"; then delgroup $username fi } diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 9131c2378..6431ccbfb 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -155,7 +155,6 @@ fi # Requires YunoHost version 2.6.4 or higher. ynh_setup_source() { # Declare an array to define the options of this helper. - local legacy_args=dsk local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) local dest_dir local source_id @@ -186,8 +185,8 @@ ynh_setup_source() { local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" - [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" - [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" + [[ -n "$src_url" ]] || ynh_die --message="No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die --message="No sha256 sum defined for source $source_id$arch_prefix ?" if [[ -z "$src_format" ]] then @@ -233,7 +232,7 @@ ynh_setup_source() { if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] then - ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" + ynh_die --message="For source $source_id, expected either 'true' or 'false' for the extract parameter" fi @@ -251,7 +250,7 @@ ynh_setup_source() { cp $local_src $src_filename fi - [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" + [ -n "$src_url" ] || ynh_die --message="Couldn't parse SOURCE_URL from $src_file_path ?" # If the file was prefetched but somehow doesn't match the sum, rm and redownload it if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status @@ -436,14 +435,14 @@ ynh_local_curl() { # Temporarily enable visitors if needed... local visitors_enabled=$(ynh_permission_has_user "main" "visitors" && echo yes || echo no) if [[ $visitors_enabled == "no" ]]; then - ynh_permission_update --permission "main" --add "visitors" + ynh_permission_update --permission="main" --add="visitors" fi # Curl the URL curl --silent --show-error --insecure --location --header "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" --cookie-jar $cookiefile --cookie $cookiefile if [[ $visitors_enabled == "no" ]]; then - ynh_permission_update --permission "main" --remove "visitors" + ynh_permission_update --permission="main" --remove="visitors" fi } @@ -489,7 +488,6 @@ ynh_local_curl() { # Requires YunoHost version 4.1.0 or higher. ynh_add_config() { # Declare an array to define the options of this helper. - local legacy_args=tdv local -A args_array=([t]=template= [d]=destination=) local template local destination @@ -550,7 +548,6 @@ ynh_add_config() { # Requires YunoHost version 4.1.0 or higher. ynh_replace_vars() { # Declare an array to define the options of this helper. - local legacy_args=f local -A args_array=([f]=file=) local file # Manage arguments with getopts @@ -647,7 +644,6 @@ ynh_replace_vars() { # Requires YunoHost version 4.3 or higher. ynh_read_var_in_file() { # Declare an array to define the options of this helper. - local legacy_args=fka local -A args_array=([f]=file= [k]=key= [a]=after=) local file local key @@ -725,7 +721,6 @@ ynh_read_var_in_file() { # Requires YunoHost version 4.3 or higher. ynh_write_var_in_file() { # Declare an array to define the options of this helper. - local legacy_args=fkva local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) local file local key @@ -873,7 +868,6 @@ _acceptable_path_to_delete() { # Requires YunoHost version 2.6.4 or higher. ynh_secure_remove() { # Declare an array to define the options of this helper. - local legacy_args=f local -A args_array=([f]=file=) local file # Manage arguments with getopts @@ -907,7 +901,6 @@ ynh_secure_remove() { # Requires YunoHost version 3.5.0 or higher. ynh_read_manifest() { # Declare an array to define the options of this helper. - local legacy_args=mk local -A args_array=([m]=manifest= [k]=manifest_key=) local manifest local manifest_key @@ -923,7 +916,7 @@ ynh_read_manifest() { then manifest="$YNH_APP_BASEDIR/manifest.toml" else - ynh_die --message "No manifest found !?" + ynh_die --message="No manifest found !?" fi fi @@ -950,7 +943,6 @@ ynh_read_manifest() { # Requires YunoHost version 3.5.0 or higher. ynh_app_upstream_version() { # Declare an array to define the options of this helper. - local legacy_args=m local -A args_array=([m]=manifest=) local manifest # Manage arguments with getopts @@ -981,7 +973,6 @@ ynh_app_upstream_version() { # Requires YunoHost version 3.5.0 or higher. ynh_app_package_version() { # Declare an array to define the options of this helper. - local legacy_args=m local -A args_array=([m]=manifest=) local manifest # Manage arguments with getopts @@ -1031,7 +1022,6 @@ ynh_check_app_version_changed() { # # Requires YunoHost version 3.8.0 or higher. ynh_compare_current_package_version() { - local legacy_args=cv declare -Ar args_array=([c]=comparison= [v]=version=) local version local comparison @@ -1077,7 +1067,7 @@ _ynh_apply_default_permissions() { chmod o-rwx $target chmod g-w $target chown -R root:root $target - if ynh_system_user_exists $app; then + if ynh_system_user_exists --username=$app; then chown $app:$app $target fi From 6e13a4db1ba5728327c4ca916ea9e05118a0453d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 18:37:38 +0200 Subject: [PATCH 005/146] helpers 2.1: remove unecessary --app=$app in internals ynh_app_setting_get/set calls --- helpers/helpers.v2.1.d/apps | 14 +++++++------- helpers/helpers.v2.1.d/apt | 8 ++++---- helpers/helpers.v2.1.d/backup | 10 +++++----- helpers/helpers.v2.1.d/config | 4 ++-- helpers/helpers.v2.1.d/go | 8 ++++---- helpers/helpers.v2.1.d/mongodb | 4 ++-- helpers/helpers.v2.1.d/mysql | 2 +- helpers/helpers.v2.1.d/nodejs | 6 +++--- helpers/helpers.v2.1.d/php | 26 +++++++++++++------------- helpers/helpers.v2.1.d/postgresql | 2 +- helpers/helpers.v2.1.d/ruby | 8 ++++---- 11 files changed, 46 insertions(+), 46 deletions(-) diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps index f0c856a23..2c6e95b37 100644 --- a/helpers/helpers.v2.1.d/apps +++ b/helpers/helpers.v2.1.d/apps @@ -51,7 +51,7 @@ ynh_install_apps() { fi done - ynh_app_setting_set --app=$app --key=apps_dependencies --value="$apps_dependencies" + ynh_app_setting_set --key=apps_dependencies --value="$apps_dependencies" } # Remove other YunoHost apps @@ -63,8 +63,8 @@ ynh_install_apps() { # Requires YunoHost version *.*.* or higher. ynh_remove_apps() { # Retrieve the apps dependencies of the app - local apps_dependencies=$(ynh_app_setting_get --app=$app --key=apps_dependencies) - ynh_app_setting_delete --app=$app --key=apps_dependencies + local apps_dependencies=$(ynh_app_setting_get --key=apps_dependencies) + ynh_app_setting_delete --key=apps_dependencies if [ ! -z "$apps_dependencies" ] then @@ -153,7 +153,7 @@ ynh_spawn_app_shell() { fi # Make sure the app has an install_dir setting - local install_dir=$(ynh_app_setting_get --app=$app --key=install_dir) + local install_dir=$(ynh_app_setting_get --key=install_dir) if [ -z "$install_dir" ] then ynh_print_err --message="$app has no install_dir setting (does it use packaging format >=2?)" @@ -161,7 +161,7 @@ ynh_spawn_app_shell() { fi # Load the app's service name, or default to $app - local service=$(ynh_app_setting_get --app=$app --key=service) + local service=$(ynh_app_setting_get --key=service) [ -z "$service" ] && service=$app; # Export HOME variable @@ -173,8 +173,8 @@ ynh_spawn_app_shell() { # Force `php` to its intended version # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` - local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) - local phpflags=$(ynh_app_setting_get --app=$app --key=phpflags) + local phpversion=$(ynh_app_setting_get --key=phpversion) + local phpflags=$(ynh_app_setting_get --key=phpflags) if [ -n "$phpversion" ] then eval "php() { php${phpversion} ${phpflags} \"\$@\"; }" diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 51a339544..ca792979d 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -266,11 +266,11 @@ ynh_install_app_dependencies() { dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common" - local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + local old_phpversion=$(ynh_app_setting_get --key=phpversion) # If the PHP version changed, remove the old fpm conf if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$specific_php_version" ]; then - local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" if [[ -f "$old_php_finalphpconf" ]] @@ -280,7 +280,7 @@ ynh_install_app_dependencies() { fi fi # Store phpversion into the config of this app - ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version + ynh_app_setting_set --key=phpversion --value=$specific_php_version # Set the default php version back as the default version for php-cli. if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION @@ -288,7 +288,7 @@ ynh_install_app_dependencies() { update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION fi elif grep --quiet 'php' <<< "$dependencies"; then - ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION + ynh_app_setting_set --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION fi local psql_installed="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)" diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index f08495923..b3834963f 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -77,7 +77,7 @@ ynh_backup() { not_mandatory="${not_mandatory:-0}" BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} - test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data) + test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --key=do_not_backup_data) # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items @@ -315,14 +315,14 @@ ynh_store_file_checksum() { # If update only, we don't save the new checksum if no old checksum exist if [ $update_only -eq 1 ]; then - local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) if [ -z "${checksum_value}" ]; then unset backup_file_checksum return 0 fi fi - ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) + ynh_app_setting_set --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ... @@ -359,7 +359,7 @@ ynh_backup_if_checksum_is_different() { ynh_handle_getopts_args "$@" local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum backup_file_checksum="" if [ -n "$checksum_value" ]; then # Proceed only if a value was stored into the app settings @@ -398,7 +398,7 @@ ynh_delete_file_checksum() { ynh_handle_getopts_args "$@" local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - ynh_app_setting_delete --app=$app --key=$checksum_setting_name + ynh_app_setting_delete --key=$checksum_setting_name } # Checks a backup archive exists diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index 6fdc74cd9..a1784aff2 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -90,7 +90,7 @@ _ynh_app_config_apply_one() { # Save value in app settings elif [[ "$bind" == "settings" ]]; then - ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_app_setting_set --key=$short_setting --value="${!short_setting}" ynh_print_info --message="Configuration key '$short_setting' edited in app settings" # Save multiline text in a file @@ -120,7 +120,7 @@ _ynh_app_config_apply_one() { ynh_store_file_checksum --file="$bind_file" --update_only # We stored the info in settings in order to be able to upgrade the app - ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_app_setting_set --key=$short_setting --value="${!short_setting}" ynh_print_info --message="Configuration key '$bind_key_' edited into $bind_file" fi diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index c4b0643c8..370f5541b 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -52,7 +52,7 @@ export GOENV_ROOT="$goenv_install_dir" # # Requires YunoHost version 3.2.2 or higher. ynh_use_go () { - go_version=$(ynh_app_setting_get --app=$app --key=go_version) + go_version=$(ynh_app_setting_get --key=go_version) # Get the absolute path of this version of Go go_path="$go_version_path/$go_version/bin" @@ -166,7 +166,7 @@ ynh_install_go () { goenv install --skip-existing $final_go_version # Store go_version into the config of this app - ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=go_version --value=$final_go_version + ynh_app_setting_set --key=go_version --value=$final_go_version # Cleanup Go versions ynh_cleanup_go @@ -188,7 +188,7 @@ eval \"\$(goenv init -)\" # # usage: ynh_remove_go ynh_remove_go () { - local go_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=go_version) + local go_version=$(ynh_app_setting_get --key=go_version) # Load goenv path in PATH local CLEAR_PATH="$goenv_install_dir/bin:$PATH" @@ -197,7 +197,7 @@ ynh_remove_go () { PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') # Remove the line for this app - ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=go_version + ynh_app_setting_delete --key=go_version # Cleanup Go versions ynh_cleanup_go diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index c8b016e1b..b81050462 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -255,7 +255,7 @@ ynh_mongo_setup_db() { ynh_mongo_create_user --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name" # Store the password in the app's config - ynh_app_setting_set --app=$app --key=db_pwd --value=$db_pwd + ynh_app_setting_set --key=db_pwd --value=$db_pwd } # Remove a database if it exists, and the associated user @@ -321,7 +321,7 @@ ynh_install_mongo() { yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" # Store mongo_version into the config of this app - ynh_app_setting_set --app=$app --key=mongo_version --value=$mongo_version + ynh_app_setting_set --key=mongo_version --value=$mongo_version } # Remove MongoDB diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index 7b7e0fd31..373bc6559 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -212,7 +212,7 @@ ynh_mysql_setup_db() { dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn --message="Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" - ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd + ynh_app_setting_set --key=mysqlpwd --value=$db_pwd } # Remove a database if it exists, and the associated user diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index 4bf9a0b71..e158affec 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -50,7 +50,7 @@ export N_PREFIX="$n_install_dir" # # Requires YunoHost version 2.7.12 or higher. ynh_use_nodejs() { - nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version) + nodejs_version=$(ynh_app_setting_get --key=nodejs_version) # Get the absolute path of this version of node nodejs_path="$node_version_path/$nodejs_version/bin" @@ -146,7 +146,7 @@ ynh_install_nodejs() { echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version" # Store nodejs_version into the config of this app - ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version + ynh_app_setting_set --key=nodejs_version --value=$nodejs_version # Build the update script and set the cronjob ynh_cron_upgrade_node @@ -164,7 +164,7 @@ ynh_install_nodejs() { # # Requires YunoHost version 2.7.12 or higher. ynh_remove_nodejs() { - nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version) + nodejs_version=$(ynh_app_setting_get --key=nodejs_version) # Remove the line for this app sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version" diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 52100560d..fa1d3a4c2 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -87,19 +87,19 @@ ynh_add_fpm_config() { autogenconf=true # If no usage provided, default to the value existing in setting ... or to low - local fpm_usage_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_usage) + local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) if [ -z "$usage" ] then usage=${fpm_usage_in_setting:-low} - ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage + ynh_app_setting_set --key=fpm_usage --value=$usage fi # If no footprint provided, default to the value existing in setting ... or to low - local fpm_footprint_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_footprint) + local fpm_footprint_in_setting=$(ynh_app_setting_get --key=fpm_footprint) if [ -z "$footprint" ] then footprint=${fpm_footprint_in_setting:-low} - ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint + ynh_app_setting_set --key=fpm_footprint --value=$footprint fi fi @@ -111,12 +111,12 @@ ynh_add_fpm_config() { phpversion="${phpversion:-$_globalphpversion}" fi - local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + local old_phpversion=$(ynh_app_setting_get --key=phpversion) # If the PHP version changed, remove the old fpm conf # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper) if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then - local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" if [[ -f "$old_php_finalphpconf" ]] @@ -132,9 +132,9 @@ ynh_add_fpm_config() { # Create the directory for FPM pools mkdir --parents "$fpm_config_dir/pool.d" - ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" - ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion + ynh_app_setting_set --key=fpm_config_dir --value="$fpm_config_dir" + ynh_app_setting_set --key=fpm_service --value="$fpm_service" + ynh_app_setting_set --key=phpversion --value=$phpversion if [ $autogenconf == "false" ]; then # Usage 1, use the template in conf/php-fpm.conf @@ -204,10 +204,10 @@ pm.process_idle_timeout = 10s # # Requires YunoHost version 2.7.2 or higher. ynh_remove_fpm_config() { - local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) - local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) + local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) + local fpm_service=$(ynh_app_setting_get --key=fpm_service) # Get the version of PHP used by this app - local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + local phpversion=$(ynh_app_setting_get --key=phpversion) # Assume default PHP-FPM version by default phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}" @@ -314,7 +314,7 @@ ynh_get_scalable_phpfpm() { fi # Get a potential forced value for php_max_children - local php_forced_max_children=$(ynh_app_setting_get --app=$app --key=php_forced_max_children) + local php_forced_max_children=$(ynh_app_setting_get --key=php_forced_max_children) if [ -n "$php_forced_max_children" ]; then php_max_children=$php_forced_max_children fi diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 09c906cb6..d3a80021a 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -249,7 +249,7 @@ ynh_psql_setup_db() { fi ynh_psql_create_db "$db_name" "$db_user" # Create the database - ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config + ynh_app_setting_set --key=psqlpwd --value=$db_pwd # Store the password in the app's config } # Remove a database if it exists, and the associated user diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 10e450ed9..2b10ec618 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -51,7 +51,7 @@ fi # # Requires YunoHost version 3.2.2 or higher. ynh_use_ruby () { - ruby_version=$(ynh_app_setting_get --app=$app --key=ruby_version) + ruby_version=$(ynh_app_setting_get --key=ruby_version) # Get the absolute path of this version of Ruby ruby_path="$ruby_version_path/$YNH_APP_INSTANCE_NAME/bin" @@ -206,7 +206,7 @@ ynh_install_ruby () { RUBY_CONFIGURE_OPTS="--disable-install-doc --with-jemalloc" MAKE_OPTS="-j2" rbenv install --skip-existing $final_ruby_version > /dev/null 2>&1 # Store ruby_version into the config of this app - ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=ruby_version --value=$final_ruby_version + ynh_app_setting_set --key=ruby_version --value=$final_ruby_version # Remove app virtualenv if rbenv alias --list | grep --quiet "$YNH_APP_INSTANCE_NAME " @@ -237,7 +237,7 @@ eval \"\$(rbenv init -)\" # # usage: ynh_remove_ruby ynh_remove_ruby () { - local ruby_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=ruby_version) + local ruby_version=$(ynh_app_setting_get --key=ruby_version) # Load rbenv path in PATH local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" @@ -248,7 +248,7 @@ ynh_remove_ruby () { rbenv alias $YNH_APP_INSTANCE_NAME --remove # Remove the line for this app - ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=ruby_version + ynh_app_setting_delete --key=ruby_version # Cleanup Ruby versions ynh_cleanup_ruby From 4a74a7c51dd54be40d26cfd7cbb67645b4f71b7d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 18:40:18 +0200 Subject: [PATCH 006/146] helpers 2.1: remove support for __NAME__ and __NAMETOCHANGE__ replaced by $app in templates --- helpers/helpers.v2.1.d/utils | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 6431ccbfb..787e51d06 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -464,8 +464,6 @@ ynh_local_curl() { # that should be defined before calling this helper : # ``` # __PATH__ by $path_url -# __NAME__ by $app -# __NAMETOCHANGE__ by $app # __USER__ by $app # __FINALPATH__ by $final_path # __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) @@ -532,8 +530,6 @@ ynh_add_config() { # The helper will replace the following keywords with global variables # that should be defined before calling this helper : # __PATH__ by $path_url -# __NAME__ by $app -# __NAMETOCHANGE__ by $app # __USER__ by $app # __FINALPATH__ by $final_path # __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) @@ -561,19 +557,8 @@ ynh_replace_vars() { ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" fi if test -n "${app:-}"; then - ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$file" - ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file" ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" fi - # Legacy - if test -n "${final_path:-}"; then - ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file" - ynh_replace_string --match_string="__INSTALL_DIR__" --replace_string="$final_path" --target_file="$file" - fi - # Legacy / Packaging v1 only - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2 && test -n "${YNH_PHP_VERSION:-}"; then - ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file" - fi if test -n "${ynh_node_load_PATH:-}"; then ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" fi From 0915a6a70b61a0a7d28be50ba73ea94e49049286 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 18:45:54 +0200 Subject: [PATCH 007/146] helpers 2.1: Drop support for old .src format in ynh_setup_source --- helpers/helpers.v2.1.d/utils | 113 ++++++++++++----------------------- 1 file changed, 38 insertions(+), 75 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 787e51d06..4605a8dc2 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -72,7 +72,7 @@ then ynh_abort_if_errors fi -# Download, check integrity, uncompress and patch the source from app.src +# Download, check integrity, uncompress and patch upstream sources # # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] # | arg: -d, --dest_dir= - Directory where to setup sources @@ -126,23 +126,6 @@ fi # # In which case ynh_setup_source --dest_dir="$install_dir" will automatically pick the appropriate source depending on the arch # -# -# -# #### Legacy format '.src' -# -# This helper will read `conf/${source_id}.src`, download and install the sources. -# -# The src file need to contains: -# ``` -# SOURCE_URL=Address to download the app archive -# SOURCE_SUM=Sha256 sum -# SOURCE_FORMAT=tar.gz -# SOURCE_IN_SUBDIR=false -# SOURCE_FILENAME=example.tar.gz -# SOURCE_EXTRACT=(true|false) -# SOURCE_PLATFORM=linux/arm64/v8 -# ``` -# # The helper will: # - Download the specific URL if there is no local archive # - Check the integrity with the specific sha256 sum @@ -165,67 +148,48 @@ ynh_setup_source() { keep="${keep:-}" full_replace="${full_replace:-0}" - if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null + source_id="${source_id:-main}" + local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") + if jq -re ".url" <<< "$sources_json" then - source_id="${source_id:-main}" - local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") - if jq -re ".url" <<< "$sources_json" - then - local arch_prefix="" - else - local arch_prefix=".$YNH_ARCH" - fi - - local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" - local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" - local src_sumprg="sha256sum" - local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" - local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" - local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" - local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" - local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" - - [[ -n "$src_url" ]] || ynh_die --message="No URL defined for source $source_id$arch_prefix ?" - [[ -n "$src_sum" ]] || ynh_die --message="No sha256 sum defined for source $source_id$arch_prefix ?" - - if [[ -z "$src_format" ]] - then - if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] - then - src_format="zip" - elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] - then - src_format="tar.gz" - elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] - then - src_format="tar.xz" - elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] - then - src_format="tar.bz2" - elif [[ -z "$src_extract" ]] - then - src_extract="false" - fi - fi + local arch_prefix="" else - source_id="${source_id:-app}" - local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" - - # Load value from configuration file (see above for a small doc about this file - # format) - local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_rename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_platform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local arch_prefix=".$YNH_ARCH" fi - # Default value - src_sumprg=${src_sumprg:-sha256sum} + local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" + local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" + local src_sumprg="sha256sum" + local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" + local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" src_in_subdir=${src_in_subdir:-true} + local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" + local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" + local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" + + [[ -n "$src_url" ]] || ynh_die --message="No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die --message="No sha256 sum defined for source $source_id$arch_prefix ?" + + if [[ -z "$src_format" ]] + then + if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] + then + src_format="zip" + elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] + then + src_format="tar.gz" + elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] + then + src_format="tar.xz" + elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] + then + src_format="tar.bz2" + elif [[ -z "$src_extract" ]] + then + src_extract="false" + fi + fi + src_format=${src_format:-tar.gz} src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') src_extract=${src_extract:-true} @@ -235,7 +199,6 @@ ynh_setup_source() { ynh_die --message="For source $source_id, expected either 'true' or 'false' for the extract parameter" fi - # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" From aa6634fd22b17d53f43f906d11b36fee7439d559 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 20:14:39 +0200 Subject: [PATCH 008/146] helpers 2.1: rework argument parsing comment to improve readability, consistency --- helpers/helpers.v2.1.d/apps | 8 +++--- helpers/helpers.v2.1.d/apt | 28 ++++++++++---------- helpers/helpers.v2.1.d/backup | 21 +++++++-------- helpers/helpers.v2.1.d/fail2ban | 4 +-- helpers/helpers.v2.1.d/go | 4 +-- helpers/helpers.v2.1.d/hardware | 8 +++--- helpers/helpers.v2.1.d/logging | 24 +++++++++-------- helpers/helpers.v2.1.d/logrotate | 3 ++- helpers/helpers.v2.1.d/mongodb | 40 ++++++++++++++-------------- helpers/helpers.v2.1.d/multimedia | 11 ++++---- helpers/helpers.v2.1.d/mysql | 28 ++++++++++---------- helpers/helpers.v2.1.d/network | 20 +++++++------- helpers/helpers.v2.1.d/nodejs | 4 +-- helpers/helpers.v2.1.d/permission | 19 +++++++++----- helpers/helpers.v2.1.d/php | 19 +++++++------- helpers/helpers.v2.1.d/postgresql | 32 +++++++++++------------ helpers/helpers.v2.1.d/ruby | 4 +-- helpers/helpers.v2.1.d/setting | 20 +++++++------- helpers/helpers.v2.1.d/string | 20 +++++++------- helpers/helpers.v2.1.d/systemd | 12 ++++----- helpers/helpers.v2.1.d/user | 25 +++++++++--------- helpers/helpers.v2.1.d/utils | 43 ++++++++++++++++--------------- 22 files changed, 202 insertions(+), 195 deletions(-) diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps index 2c6e95b37..e4d7f90ca 100644 --- a/helpers/helpers.v2.1.d/apps +++ b/helpers/helpers.v2.1.d/apps @@ -7,11 +7,11 @@ # # Requires YunoHost version *.*.* or higher. ynh_install_apps() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([a]=apps=) local apps - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Split the list of apps in an array local apps_list=($(echo $apps | tr " " "\n")) @@ -125,11 +125,11 @@ ynh_remove_apps() { # from the app's service configuration file (defaults to $app.service, overridable by the packager with `service` setting). # If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings. ynh_spawn_app_shell() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([a]=app=) local app - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Force Bash to be used to run this helper if [[ ! $0 =~ \/?bash$ ]] diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index ca792979d..2553cdc03 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -68,11 +68,11 @@ ynh_package_is_installed() { # # Requires YunoHost version 2.2.4 or higher. ynh_package_version() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=package=) local package - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if ynh_package_is_installed "$package"; then dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null @@ -344,11 +344,11 @@ EOF # # Requires YunoHost version 3.8.1 or higher. ynh_add_app_dependencies() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=package= [r]=replace) local package - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== ynh_print_warn --message="Packagers: ynh_add_app_dependencies is deprecated and is now only an alias to ynh_install_app_dependencies" ynh_install_app_dependencies "${package}" @@ -400,16 +400,16 @@ ynh_remove_app_dependencies() { # # Requires YunoHost version 3.8.1 or higher. ynh_install_extra_app_dependencies() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([r]=repo= [p]=package= [k]=key= [n]=name=) local repo local package local key local name - # Manage arguments with getopts ynh_handle_getopts_args "$@" name="${name:-$app}" key=${key:-} + # =========================================== # Set a key only if asked if [ -n "$key" ]; then @@ -444,19 +444,19 @@ ynh_install_extra_app_dependencies() { # # Requires YunoHost version 3.8.1 or higher. ynh_install_extra_repo() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append) local repo local key local priority local name local append - # Manage arguments with getopts ynh_handle_getopts_args "$@" name="${name:-$app}" append=${append:-0} key=${key:-} priority=${priority:-} + # =========================================== if [ $append -eq 1 ]; then append="--append" @@ -512,12 +512,12 @@ ynh_install_extra_repo() { # # Requires YunoHost version 3.8.1 or higher. ynh_remove_extra_repo() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([n]=name=) local name - # Manage arguments with getopts ynh_handle_getopts_args "$@" name="${name:-$app}" + # =========================================== ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list" # Sury pinning is managed by the regenconf in the core... @@ -552,17 +552,17 @@ ynh_remove_extra_repo() { # # Requires YunoHost version 3.8.1 or higher. ynh_add_repo() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append) local uri local suite local component local name local append - # Manage arguments with getopts ynh_handle_getopts_args "$@" name="${name:-$app}" append=${append:-0} + # =========================================== if [ $append -eq 1 ]; then append="tee --append" @@ -591,19 +591,19 @@ ynh_add_repo() { # # Requires YunoHost version 3.8.1 or higher. ynh_pin_repo() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append) local package local pin local priority local name local append - # Manage arguments with getopts ynh_handle_getopts_args "$@" package="${package:-*}" priority=${priority:-50} name="${name:-$app}" append=${append:-0} + # =========================================== if [ $append -eq 1 ]; then append="tee --append" diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index b3834963f..820a2562d 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -64,17 +64,17 @@ CAN_BIND=${CAN_BIND:-1} ynh_backup() { # TODO find a way to avoid injection by file strange naming ! - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory) local src_path local dest_path local is_big local not_mandatory - # Manage arguments with getopts ynh_handle_getopts_args "$@" dest_path="${dest_path:-}" is_big="${is_big:-0}" not_mandatory="${not_mandatory:-0}" + # =========================================== BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --key=do_not_backup_data) @@ -235,17 +235,17 @@ with open(sys.argv[1], 'r') as backup_file: # Requires YunoHost version 2.6.4 or higher. # Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory ynh_restore_file() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory) local origin_path local dest_path local not_mandatory - # Manage arguments with getopts ynh_handle_getopts_args "$@" origin_path="/${origin_path#/}" # Default value for dest_path = /$origin_path dest_path="${dest_path:-$origin_path}" not_mandatory="${not_mandatory:-0}" + # =========================================== local archive_path="$YNH_CWD${origin_path}" # If archive_path doesn't exist, search for a corresponding path in CSV @@ -302,14 +302,13 @@ ynh_restore_file() { # # Requires YunoHost version 2.6.4 or higher. ynh_store_file_checksum() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file= [u]=update_only) local file local update_only update_only="${update_only:-0}" - - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' @@ -352,11 +351,11 @@ ynh_store_file_checksum() { # # Requires YunoHost version 2.6.4 or higher. ynh_backup_if_checksum_is_different() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file=) local file - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) @@ -391,11 +390,11 @@ ynh_backup_if_checksum_is_different() { # # Requires YunoHost version 3.3.1 or higher. ynh_delete_file_checksum() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file=) local file - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' ynh_app_setting_delete --key=$checksum_setting_name diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index 9343d2bea..12c579db6 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -58,18 +58,18 @@ # # Requires YunoHost version 4.1.0 or higher. ynh_add_fail2ban_config() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template) local logpath local failregex local max_retry local ports local use_template - # Manage arguments with getopts ynh_handle_getopts_args "$@" max_retry=${max_retry:-3} ports=${ports:-http,https} use_template="${use_template:-0}" + # =========================================== if [ "$use_template" -ne 1 ]; then # Usage 1, no template. Build a config file from scratch. diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 370f5541b..cf8290390 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -94,11 +94,11 @@ ynh_use_go () { # # Requires YunoHost version 3.2.2 or higher. ynh_install_go () { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [v]=go_version= ) local go_version - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Load goenv path in PATH local CLEAR_PATH="$goenv_install_dir/bin:$PATH" diff --git a/helpers/helpers.v2.1.d/hardware b/helpers/helpers.v2.1.d/hardware index aed1a18ba..678f381b1 100644 --- a/helpers/helpers.v2.1.d/hardware +++ b/helpers/helpers.v2.1.d/hardware @@ -13,18 +13,18 @@ # # Requires YunoHost version 3.8.1 or higher. ynh_get_ram() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=free [t]=total [s]=ignore_swap [o]=only_swap) local free local total local ignore_swap local only_swap - # Manage arguments with getopts ynh_handle_getopts_args "$@" ignore_swap=${ignore_swap:-0} only_swap=${only_swap:-0} free=${free:-0} total=${total:-0} + # =========================================== if [ $free -eq $total ]; then ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram" @@ -76,14 +76,13 @@ ynh_get_ram() { # # Requires YunoHost version 3.8.1 or higher. ynh_require_ram() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap) local required local free local total local ignore_swap local only_swap - # Manage arguments with getopts ynh_handle_getopts_args "$@" # Dunno if that's the right way to do, but that's some black magic to be able to # forward the bool args to ynh_get_ram easily? @@ -92,6 +91,7 @@ ynh_require_ram() { total=${total:+--total} ignore_swap=${ignore_swap:+--ignore_swap} only_swap=${only_swap:+--only_swap} + # =========================================== local ram=$(ynh_get_ram $free $total $ignore_swap $only_swap) diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index d761a20e5..1e4b15871 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -8,13 +8,13 @@ # # Requires YunoHost version 2.4.0 or higher. ynh_die() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=message= [c]=ret_code=) local message local ret_code - # Manage arguments with getopts ynh_handle_getopts_args "$@" ret_code=${ret_code:-1} + # =========================================== echo "$message" 1>&2 exit "$ret_code" @@ -27,11 +27,11 @@ ynh_die() { # # Requires YunoHost version 3.2.0 or higher. ynh_print_info() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=message=) local message - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== echo "$message" >&$YNH_STDINFO } @@ -52,11 +52,11 @@ ynh_print_log() { # # Requires YunoHost version 3.2.0 or higher. ynh_print_warn() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=message=) local message - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== ynh_print_log "${message}" >&2 } @@ -68,11 +68,11 @@ ynh_print_warn() { # # Requires YunoHost version 3.2.0 or higher. ynh_print_err() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=message=) local message - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== ynh_print_log "[Error] ${message}" >&2 } @@ -252,17 +252,19 @@ base_time=$(date +%s) # Requires YunoHost version 3.5.0 or higher. ynh_script_progression() { set +o xtrace # set +x - # Declare an array to define the options of this helper. + + # ============ Argument parsing ============= local -A args_array=([m]=message= [w]=weight= [t]=time [l]=last) local message local weight local time local last - # Manage arguments with getopts ynh_handle_getopts_args "$@" + weight=${weight:-1} + # =========================================== + # Re-disable xtrace, ynh_handle_getopts_args set it back set +o xtrace # set +x - weight=${weight:-1} # Always activate time when running inside CI tests if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index 4f5cb69e9..970377acf 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -26,13 +26,14 @@ ynh_use_logrotate() { done set -- "${all_args[@]}" - # Argument parsing + # ============ Argument parsing ============= local -A args_array=([l]=logfile= [u]=specific_user=) local logfile local specific_user ynh_handle_getopts_args "$@" logfile="${logfile:-}" specific_user="${specific_user:-}" + # =========================================== set -o noglob if [[ -z "$logfile" ]]; then diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index b81050462..c86369590 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -17,7 +17,7 @@ # # ynh_mongo_exec() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [u]=user= [p]=password= [a]=authenticationdatabase= [d]=database= [h]=host= [P]=port= [c]=command= [e]=eval ) local user local password @@ -27,7 +27,6 @@ ynh_mongo_exec() { local port local command local eval - # Manage arguments with getopts ynh_handle_getopts_args "$@" user="${user:-}" password="${password:-}" @@ -36,6 +35,7 @@ ynh_mongo_exec() { host="${host:-}" port="${port:-}" eval=${eval:-0} + # =========================================== # If user is provided if [ -n "$user" ] @@ -113,11 +113,11 @@ EOF # # ynh_mongo_drop_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [d]=database= ) local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== ynh_mongo_exec --database="$database" --command='db.runCommand({dropDatabase: 1})' } @@ -132,11 +132,11 @@ ynh_mongo_drop_db() { # # ynh_mongo_dump_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [d]=database= ) local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== mongodump --quiet --db="$database" --archive } @@ -152,13 +152,13 @@ ynh_mongo_dump_db() { # # ynh_mongo_create_user() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) local db_user local db_name local db_pwd - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Create the user and set the user as admin of the db ynh_mongo_exec --database="$db_name" --command='db.createUser( { user: "'${db_user}'", pwd: "'${db_pwd}'", roles: [ { role: "readWrite", db: "'${db_name}'" } ] } );' @@ -175,11 +175,11 @@ ynh_mongo_create_user() { # # ynh_mongo_database_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([d]=database=) local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if [ $(ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("'${database}'")' --eval) -lt 0 ] then @@ -198,11 +198,11 @@ ynh_mongo_database_exists() { # # ynh_mongo_restore_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [d]=database= ) local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== mongorestore --quiet --db="$database" --archive } @@ -217,12 +217,12 @@ ynh_mongo_restore_db() { # # ynh_mongo_drop_user() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [u]=db_user= [n]=db_name= ) local db_user local db_name - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== ynh_mongo_exec --database="$db_name" --command='db.dropUser("'$db_user'", {w: "majority", wtimeout: 5000})' } @@ -239,13 +239,13 @@ ynh_mongo_drop_user() { # # ynh_mongo_setup_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) local db_user local db_name db_pwd="" - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== local new_db_pwd=$(ynh_string_random) # Generate a random password # If $db_pwd is not provided, use new_db_pwd instead for db_pwd @@ -266,12 +266,12 @@ ynh_mongo_setup_db() { # # ynh_mongo_remove_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [u]=db_user= [n]=db_name= ) local db_user local db_name - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists ynh_mongo_drop_db --database=$db_name # Remove the database @@ -290,12 +290,12 @@ ynh_mongo_remove_db() { # # ynh_install_mongo() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=mongo_version=) local mongo_version - # Manage arguments with getopts ynh_handle_getopts_args "$@" mongo_version="${mongo_version:-$YNH_MONGO_VERSION}" + # =========================================== ynh_print_info --message="Installing MongoDB Community Edition ..." local mongo_debian_release=$(ynh_get_debian_release) diff --git a/helpers/helpers.v2.1.d/multimedia b/helpers/helpers.v2.1.d/multimedia index b1d8a526e..16f085468 100644 --- a/helpers/helpers.v2.1.d/multimedia +++ b/helpers/helpers.v2.1.d/multimedia @@ -63,12 +63,12 @@ ynh_multimedia_build_main_dir() { # Requires YunoHost version 4.2 or higher. ynh_multimedia_addfolder() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([s]=source_dir= [d]=dest_dir=) local source_dir local dest_dir - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Ajout d'un lien symbolique vers le dossier à partager ln -sfn "$source_dir" "$MEDIA_DIRECTORY/$dest_dir" @@ -90,11 +90,12 @@ ynh_multimedia_addfolder() { # # Requires YunoHost version 4.2 or higher. ynh_multimedia_addaccess() { - # Declare an array to define the options of this helper. - declare -Ar args_array=([u]=user_name=) + + # ============ Argument parsing ============= + local -A args_array=([u]=user_name=) local user_name - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== groupadd -f multimedia usermod -a -G multimedia $user_name diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index 373bc6559..1f91735ed 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -13,14 +13,14 @@ # # Requires YunoHost version 2.2.4 or higher. ynh_mysql_connect_as() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=user= [p]=password= [d]=database=) local user local password local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" database="${database:-}" + # =========================================== mysql --user="$user" --password="$password" --batch "$database" } @@ -33,13 +33,13 @@ ynh_mysql_connect_as() { # # Requires YunoHost version 2.2.4 or higher. ynh_mysql_execute_as_root() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([s]=sql= [d]=database=) local sql local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" database="${database:-}" + # =========================================== if [ -n "$database" ]; then database="--database=$database" @@ -56,13 +56,13 @@ ynh_mysql_execute_as_root() { # # Requires YunoHost version 2.2.4 or higher. ynh_mysql_execute_file_as_root() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file= [d]=database=) local file local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" database="${database:-}" + # =========================================== if [ -n "$database" ]; then database="--database=$database" @@ -123,11 +123,11 @@ ynh_mysql_drop_db() { # # Requires YunoHost version 2.2.4 or higher. ynh_mysql_dump_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([d]=database=) local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== mysqldump --single-transaction --skip-dump-date --routines "$database" } @@ -156,11 +156,11 @@ ynh_mysql_create_user() { # # Requires YunoHost version 2.2.4 or higher. ynh_mysql_user_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=user=) local user - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]; then return 1 @@ -195,13 +195,13 @@ ynh_mysql_drop_user() { # # Requires YunoHost version 2.6.4 or higher. ynh_mysql_setup_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) local db_user local db_name db_pwd="" - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Generate a random password local new_db_pwd=$(ynh_string_random) @@ -225,12 +225,12 @@ ynh_mysql_setup_db() { # # Requires YunoHost version 2.6.4 or higher. ynh_mysql_remove_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -Ar args_array=([u]=db_user= [n]=db_name=) local db_user local db_name - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if mysqlshow | grep -q "^| $db_name "; then ynh_mysql_drop_db $db_name diff --git a/helpers/helpers.v2.1.d/network b/helpers/helpers.v2.1.d/network index c407af512..c7acf2cce 100644 --- a/helpers/helpers.v2.1.d/network +++ b/helpers/helpers.v2.1.d/network @@ -12,11 +12,11 @@ # # Requires YunoHost version 2.6.4 or higher. ynh_find_port() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=port=) local port - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." while ! ynh_port_available --port=$port; do @@ -37,11 +37,11 @@ ynh_find_port() { # # Requires YunoHost version 3.8.0 or higher. ynh_port_available() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=port=) local port - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Check if the port is free if ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ":$port$"; then @@ -67,12 +67,12 @@ ynh_port_available() { ynh_validate_ip() { # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=family= [i]=ip_address=) local family local ip_address - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 @@ -98,11 +98,11 @@ EOF # # Requires YunoHost version 2.2.4 or higher. ynh_validate_ip4() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([i]=ip_address=) local ip_address - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== ynh_validate_ip --family=4 --ip_address=$ip_address } @@ -117,11 +117,11 @@ ynh_validate_ip4() { # # Requires YunoHost version 2.2.4 or higher. ynh_validate_ip6() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([i]=ip_address=) local ip_address - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== ynh_validate_ip --family=6 --ip_address=$ip_address } diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index e158affec..74491a590 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -94,11 +94,11 @@ ynh_use_nodejs() { ynh_install_nodejs() { # Use n, https://github.com/tj/n to manage the nodejs versions - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([n]=nodejs_version=) local nodejs_version - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Create $n_install_dir mkdir --parents "$n_install_dir" diff --git a/helpers/helpers.v2.1.d/permission b/helpers/helpers.v2.1.d/permission index bf2ca86ed..28847cd08 100644 --- a/helpers/helpers.v2.1.d/permission +++ b/helpers/helpers.v2.1.d/permission @@ -66,7 +66,7 @@ # # Requires YunoHost version 3.7.0 or higher. ynh_permission_create() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected=) local permission local url @@ -84,6 +84,7 @@ ynh_permission_create() { label=${label:-} show_tile=${show_tile:-} protected=${protected:-} + # =========================================== if [[ -n $url ]]; then url=",url='$url'" @@ -153,10 +154,11 @@ ynh_permission_create() { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_delete() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=permission=) local permission ynh_handle_getopts_args "$@" + # =========================================== yunohost tools shell -c "from yunohost.permission import permission_delete; permission_delete('$app.$permission')" } @@ -171,10 +173,11 @@ ynh_permission_delete() { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=permission=) local permission ynh_handle_getopts_args "$@" + # =========================================== yunohost user permission list "$app" --output-as json --quiet \ | jq -e --arg perm "$app.$permission" '.permissions[$perm]' >/dev/null @@ -195,7 +198,7 @@ ynh_permission_exists() { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_url() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls) local permission local url @@ -209,6 +212,7 @@ ynh_permission_url() { remove_url=${remove_url:-} auth_header=${auth_header:-} clear_urls=${clear_urls:-} + # =========================================== if [[ -n $url ]]; then url=",url='$url'" @@ -264,7 +268,7 @@ ynh_permission_url() { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_update() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected=) local permission local add @@ -278,6 +282,7 @@ ynh_permission_update() { label=${label:-} show_tile=${show_tile:-} protected=${protected:-} + # =========================================== if [[ -n $add ]]; then # Convert a list from getopts to python list @@ -332,12 +337,12 @@ ynh_permission_update() { # # Requires YunoHost version 3.7.1 or higher. ynh_permission_has_user() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=permission= [u]=user=) local permission local user - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if ! ynh_permission_exists --permission=$permission; then return 1 diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index fa1d3a4c2..d32e69392 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -68,16 +68,16 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # # Requires YunoHost version 4.1.0 or higher. ynh_add_fpm_config() { + # ============ Argument parsing ============= local _globalphpversion=${phpversion-:} - # Declare an array to define the options of this helper. local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [g]=group=) local group local phpversion local usage local footprint - # Manage arguments with getopts ynh_handle_getopts_args "$@" group=${group:-} + # =========================================== # The default behaviour is to use the template. local autogenconf=false @@ -242,12 +242,11 @@ ynh_remove_fpm_config() { # # | arg: -p, --print - Print the result (intended for debug purpose only when packaging the app) ynh_get_scalable_phpfpm() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=usage= [f]=footprint= [p]=print) local usage local footprint local print - # Manage arguments with getopts ynh_handle_getopts_args "$@" # Set all characters as lowercase footprint=${footprint,,} @@ -372,13 +371,12 @@ YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} # # Requires YunoHost version 4.2 or higher. ynh_composer_exec() { + # ============ Argument parsing ============= local _globalphpversion=${phpversion-:} - # Declare an array to define the options of this helper. - declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=) + local -A args_array=([v]=phpversion= [w]=workdir= [c]=commands=) local phpversion local workdir local commands - # Manage arguments with getopts ynh_handle_getopts_args "$@" workdir="${workdir:-${install_dir:-$final_path}}" @@ -403,15 +401,16 @@ ynh_composer_exec() { # # Requires YunoHost version 4.2 or higher. ynh_install_composer() { + # ============ Argument parsing ============= local _globalphpversion=${phpversion-:} - # Declare an array to define the options of this helper. - declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) + local -A args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) local phpversion local workdir local install_args local composerversion - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then workdir="${workdir:-$final_path}" else diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index d3a80021a..b28e11e38 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -16,14 +16,14 @@ PSQL_VERSION=13 # # Requires YunoHost version 3.5.0 or higher. ynh_psql_connect_as() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=user= [p]=password= [d]=database=) local user local password local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" database="${database:-}" + # =========================================== sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" } @@ -36,13 +36,13 @@ ynh_psql_connect_as() { # # Requires YunoHost version 3.5.0 or higher. ynh_psql_execute_as_root() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([s]=sql= [d]=database=) local sql local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" database="${database:-}" + # =========================================== if [ -n "$database" ]; then database="--database=$database" @@ -60,13 +60,13 @@ ynh_psql_execute_as_root() { # # Requires YunoHost version 3.5.0 or higher. ynh_psql_execute_file_as_root() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file= [d]=database=) local file local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" database="${database:-}" + # =========================================== if [ -n "$database" ]; then database="--database=$database" @@ -130,11 +130,11 @@ ynh_psql_drop_db() { # # Requires YunoHost version 3.5.0 or higher. ynh_psql_dump_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([d]=database=) local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== sudo --login --user=postgres pg_dump "$database" } @@ -164,11 +164,11 @@ ynh_psql_create_user() { # # Requires YunoHost version 3.5.0 or higher. ynh_psql_user_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=user=) local user - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user"; then return 1 @@ -185,11 +185,11 @@ ynh_psql_user_exists() { # # Requires YunoHost version 3.5.0 or higher. ynh_psql_database_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([d]=database=) local database - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # if psql is not there, we cannot check the db # though it could exists. @@ -230,13 +230,13 @@ ynh_psql_drop_user() { # # Requires YunoHost version 2.7.13 or higher. ynh_psql_setup_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) local db_user local db_name db_pwd="" - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if ! ynh_psql_user_exists --user=$db_user; then local new_db_pwd=$(ynh_string_random) # Generate a random password @@ -262,12 +262,12 @@ ynh_psql_setup_db() { # # Requires YunoHost version 2.7.13 or higher. ynh_psql_remove_db() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=db_user= [n]=db_name=) local db_user local db_name - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if ynh_psql_database_exists --database=$db_name; then # Check if the database exists ynh_psql_drop_db $db_name # Remove the database diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 2b10ec618..99fa6f016 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -96,11 +96,11 @@ ynh_use_ruby () { # # Requires YunoHost version 3.2.2 or higher. ynh_install_ruby () { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=( [v]=ruby_version= ) local ruby_version - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Load rbenv path in PATH local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index bd9aa2ab4..67c4d5f0d 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -8,14 +8,14 @@ # # Requires YunoHost version 2.2.4 or higher. ynh_app_setting_get() { + # ============ Argument parsing ============= local _globalapp=${app-:} - # Declare an array to define the options of this helper. local -A args_array=([a]=app= [k]=key=) local app local key - # Manage arguments with getopts ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" + # =========================================== ynh_app_setting "get" "$app" "$key" } @@ -29,15 +29,15 @@ ynh_app_setting_get() { # # Requires YunoHost version 2.2.4 or higher. ynh_app_setting_set() { + # ============ Argument parsing ============= local _globalapp=${app-:} - # Declare an array to define the options of this helper. local -A args_array=([a]=app= [k]=key= [v]=value=) local app local key local value - # Manage arguments with getopts ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" + # =========================================== ynh_app_setting "set" "$app" "$key" "$value" } @@ -50,14 +50,14 @@ ynh_app_setting_set() { # # Requires YunoHost version 2.2.4 or higher. ynh_app_setting_delete() { + # ============ Argument parsing ============= local _globalapp=${app-:} - # Declare an array to define the options of this helper. local -A args_array=([a]=app= [k]=key=) local app local key - # Manage arguments with getopts ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" + # =========================================== ynh_app_setting "delete" "$app" "$key" } @@ -108,12 +108,12 @@ EOF # # Requires YunoHost version 2.6.4 or higher. ynh_webpath_available() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([d]=domain= [p]=path_url=) local domain local path_url - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== yunohost domain url-available $domain $path_url } @@ -131,13 +131,13 @@ ynh_webpath_available() { # # Requires YunoHost version 2.6.4 or higher. ynh_webpath_register() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([a]=app= [d]=domain= [p]=path_url=) local app local domain local path_url - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== yunohost app register-url $app $domain $path_url } diff --git a/helpers/helpers.v2.1.d/string b/helpers/helpers.v2.1.d/string index f022deab2..398facec2 100644 --- a/helpers/helpers.v2.1.d/string +++ b/helpers/helpers.v2.1.d/string @@ -11,14 +11,14 @@ # # Requires YunoHost version 2.2.4 or higher. ynh_string_random() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([l]=length= [f]=filter=) local length local filter - # Manage arguments with getopts ynh_handle_getopts_args "$@" length=${length:-24} filter=${filter:-'A-Za-z0-9'} + # =========================================== dd if=/dev/urandom bs=1 count=1000 2>/dev/null \ | tr --complement --delete "$filter" \ @@ -37,13 +37,13 @@ ynh_string_random() { # # Requires YunoHost version 2.6.4 or higher. ynh_replace_string() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) local match_string local replace_string local target_file - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== set +o xtrace # set +x local delimit=$'\001' @@ -67,13 +67,13 @@ ynh_replace_string() { # # Requires YunoHost version 2.7.7 or higher. ynh_replace_special_string() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) local match_string local replace_string local target_file - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Escape any backslash to preserve them as simple backslash. match_string=${match_string//\\/"\\\\"} @@ -100,11 +100,11 @@ ynh_replace_special_string() { # # Requires YunoHost version 2.2.4 or higher. ynh_sanitize_dbid() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([n]=db_name=) local db_name - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # We should avoid having - and . in the name of databases. They are replaced by _ echo ${db_name//[-.]/_} @@ -129,11 +129,11 @@ ynh_sanitize_dbid() { # # Requires YunoHost version 2.6.4 or higher. ynh_normalize_url_path() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([p]=path_url=) local path_url - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 57d6b3802..d1742c23e 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -13,14 +13,14 @@ # # Requires YunoHost version 4.1.0 or higher. ynh_add_systemd_config() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([s]=service= [t]=template=) local service local template - # Manage arguments with getopts ynh_handle_getopts_args "$@" service="${service:-$app}" template="${template:-systemd.service}" + # =========================================== ynh_add_config --template="$template" --destination="/etc/systemd/system/$service.service" @@ -35,12 +35,12 @@ ynh_add_systemd_config() { # # Requires YunoHost version 2.7.2 or higher. ynh_remove_systemd_config() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([s]=service=) local service - # Manage arguments with getopts ynh_handle_getopts_args "$@" local service="${service:-$app}" + # =========================================== local finalsystemdconf="/etc/systemd/system/$service.service" if [ -e "$finalsystemdconf" ]; then @@ -63,7 +63,7 @@ ynh_remove_systemd_config() { # # Requires YunoHost version 3.5.0 or higher. ynh_systemd_action() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=) local service_name local action @@ -71,7 +71,6 @@ ynh_systemd_action() { local length local log_path local timeout - # Manage arguments with getopts ynh_handle_getopts_args "$@" service_name="${service_name:-$app}" action=${action:-start} @@ -79,6 +78,7 @@ ynh_systemd_action() { length=${length:-20} log_path="${log_path:-/var/log/$service_name/$service_name.log}" timeout=${timeout:-300} + # =========================================== # Manage case of service already stopped if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service_name; then diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index 14a8e32b5..7ab2301fa 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -10,11 +10,11 @@ # # Requires YunoHost version 2.2.4 or higher. ynh_user_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=username=) local username - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null } @@ -30,12 +30,12 @@ ynh_user_exists() { # # Requires YunoHost version 2.2.4 or higher. ynh_user_get_info() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=username= [k]=key=) local username local key - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== yunohost user info "$username" --output-as json --quiet | jq -r ".$key" } @@ -62,11 +62,11 @@ ynh_user_list() { # # Requires YunoHost version 2.2.4 or higher. ynh_system_user_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=username=) local username - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== getent passwd "$username" &>/dev/null } @@ -81,11 +81,11 @@ ynh_system_user_exists() { # # Requires YunoHost version 3.5.0.2 or higher. ynh_system_group_exists() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([g]=group=) local group - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== getent group "$group" &>/dev/null } @@ -111,18 +111,17 @@ ynh_system_group_exists() { # # Requires YunoHost version 2.6.4 or higher. ynh_system_user_create() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=username= [h]=home_dir= [s]=use_shell [g]=groups=) local username local home_dir local use_shell local groups - - # Manage arguments with getopts ynh_handle_getopts_args "$@" use_shell="${use_shell:-0}" home_dir="${home_dir:-}" groups="${groups:-}" + # =========================================== if ! ynh_system_user_exists --username="$username"; then # Check if the user exists on the system # If the user doesn't exist @@ -154,11 +153,11 @@ ynh_system_user_create() { # # Requires YunoHost version 2.6.4 or higher. ynh_system_user_delete() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([u]=username=) local username - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Check if the user exists on the system if ynh_system_user_exists --username="$username"; then diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 4605a8dc2..627124bb6 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -137,18 +137,18 @@ fi # # Requires YunoHost version 2.6.4 or higher. ynh_setup_source() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) local dest_dir local source_id local keep local full_replace - # Manage arguments with getopts ynh_handle_getopts_args "$@" keep="${keep:-}" full_replace="${full_replace:-0}" - source_id="${source_id:-main}" + # =========================================== + local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") if jq -re ".url" <<< "$sources_json" then @@ -448,13 +448,13 @@ ynh_local_curl() { # # Requires YunoHost version 4.1.0 or higher. ynh_add_config() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([t]=template= [d]=destination=) local template local destination - # Manage arguments with getopts ynh_handle_getopts_args "$@" local template_path + # =========================================== if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then template_path="$YNH_APP_BASEDIR/conf/$template" @@ -506,11 +506,11 @@ ynh_add_config() { # # Requires YunoHost version 4.1.0 or higher. ynh_replace_vars() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file=) local file - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== # Replace specific YunoHost variables if test -n "${path_url:-}"; then @@ -591,14 +591,14 @@ ynh_replace_vars() { # # Requires YunoHost version 4.3 or higher. ynh_read_var_in_file() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file= [k]=key= [a]=after=) local file local key local after - # Manage arguments with getopts ynh_handle_getopts_args "$@" after="${after:-}" + # =========================================== [[ -f $file ]] || ynh_die --message="File $file does not exists" @@ -668,15 +668,15 @@ ynh_read_var_in_file() { # # Requires YunoHost version 4.3 or higher. ynh_write_var_in_file() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) local file local key local value local after - # Manage arguments with getopts ynh_handle_getopts_args "$@" after="${after:-}" + # =========================================== [[ -f $file ]] || ynh_die --message="File $file does not exists" @@ -815,11 +815,11 @@ _acceptable_path_to_delete() { # # Requires YunoHost version 2.6.4 or higher. ynh_secure_remove() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([f]=file=) local file - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== set +o xtrace # set +x if [ $# -ge 2 ]; then @@ -848,12 +848,12 @@ ynh_secure_remove() { # # Requires YunoHost version 3.5.0 or higher. ynh_read_manifest() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=manifest= [k]=manifest_key=) local manifest local manifest_key - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== if [ ! -e "${manifest:-}" ]; then # If the manifest isn't found, try the common place for backup and restore script. @@ -890,12 +890,12 @@ ynh_read_manifest() { # # Requires YunoHost version 3.5.0 or higher. ynh_app_upstream_version() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=manifest=) local manifest - # Manage arguments with getopts ynh_handle_getopts_args "$@" manifest="${manifest:-}" + # =========================================== if [[ "$manifest" != "" ]] && [[ -e "$manifest" ]]; then version_key_=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") @@ -920,11 +920,11 @@ ynh_app_upstream_version() { # # Requires YunoHost version 3.5.0 or higher. ynh_app_package_version() { - # Declare an array to define the options of this helper. + # ============ Argument parsing ============= local -A args_array=([m]=manifest=) local manifest - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== version_key_=$YNH_APP_MANIFEST_VERSION echo "${version_key_/*~ynh/}" @@ -970,11 +970,12 @@ ynh_check_app_version_changed() { # # Requires YunoHost version 3.8.0 or higher. ynh_compare_current_package_version() { - declare -Ar args_array=([c]=comparison= [v]=version=) + # ============ Argument parsing ============= + local -A args_array=([c]=comparison= [v]=version=) local version local comparison - # Manage arguments with getopts ynh_handle_getopts_args "$@" + # =========================================== local current_version=$YNH_APP_CURRENT_VERSION From 365d0b25af9f2012621550b585d7a74614291ea8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 20:19:25 +0200 Subject: [PATCH 009/146] helpers 2.1: drop ynh_find_port, ynh_port_available, ynh_validate_ip4/6, keep only ynh_validate_ip --- helpers/helpers.v2.1.d/network | 127 --------------------------------- helpers/helpers.v2.1.d/utils | 31 ++++++++ 2 files changed, 31 insertions(+), 127 deletions(-) delete mode 100644 helpers/helpers.v2.1.d/network diff --git a/helpers/helpers.v2.1.d/network b/helpers/helpers.v2.1.d/network deleted file mode 100644 index c7acf2cce..000000000 --- a/helpers/helpers.v2.1.d/network +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -# Find a free port and return it -# -# [packagingv1] -# -# usage: ynh_find_port --port=begin_port -# | arg: -p, --port= - port to start to search -# | ret: the port number -# -# example: port=$(ynh_find_port --port=8080) -# -# Requires YunoHost version 2.6.4 or higher. -ynh_find_port() { - # ============ Argument parsing ============= - local -A args_array=([p]=port=) - local port - ynh_handle_getopts_args "$@" - # =========================================== - - test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." - while ! ynh_port_available --port=$port; do - port=$((port + 1)) - done - echo $port -} - -# Test if a port is available -# -# [packagingv1] -# -# usage: ynh_find_port --port=XYZ -# | arg: -p, --port= - port to check -# | ret: 0 if the port is available, 1 if it is already used by another process. -# -# example: ynh_port_available --port=1234 || ynh_die --message="Port 1234 is needs to be available for this app" -# -# Requires YunoHost version 3.8.0 or higher. -ynh_port_available() { - # ============ Argument parsing ============= - local -A args_array=([p]=port=) - local port - ynh_handle_getopts_args "$@" - # =========================================== - - # Check if the port is free - if ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ":$port$"; then - return 1 - # This is to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up) - elif grep -q "port: '$port'" /etc/yunohost/apps/*/settings.yml; then - return 1 - else - return 0 - fi -} - -# Validate an IP address -# -# [internal] -# -# usage: ynh_validate_ip --family=family --ip_address=ip_address -# | ret: 0 for valid ip addresses, 1 otherwise -# -# example: ynh_validate_ip 4 111.222.333.444 -# -# Requires YunoHost version 2.2.4 or higher. -ynh_validate_ip() { - # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 - - # ============ Argument parsing ============= - local -A args_array=([f]=family= [i]=ip_address=) - local family - local ip_address - ynh_handle_getopts_args "$@" - # =========================================== - - [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 - - python3 /dev/stdin < Date: Mon, 27 May 2024 20:20:12 +0200 Subject: [PATCH 010/146] helpers 2.1: fix indent --- helpers/helpers.v2.1.d/apps | 146 ++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps index e4d7f90ca..c010dab2b 100644 --- a/helpers/helpers.v2.1.d/apps +++ b/helpers/helpers.v2.1.d/apps @@ -125,89 +125,89 @@ ynh_remove_apps() { # from the app's service configuration file (defaults to $app.service, overridable by the packager with `service` setting). # If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings. ynh_spawn_app_shell() { - # ============ Argument parsing ============= - local -A args_array=([a]=app=) - local app - ynh_handle_getopts_args "$@" - # =========================================== + # ============ Argument parsing ============= + local -A args_array=([a]=app=) + local app + ynh_handle_getopts_args "$@" + # =========================================== - # Force Bash to be used to run this helper - if [[ ! $0 =~ \/?bash$ ]] - then - ynh_print_err --message="Please use Bash as shell" - exit 1 - fi + # Force Bash to be used to run this helper + if [[ ! $0 =~ \/?bash$ ]] + then + ynh_print_err --message="Please use Bash as shell" + exit 1 + fi - # Make sure the app is installed - local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id)) - if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]] - then - ynh_print_err --message="$app is not in the apps list" - exit 1 - fi + # Make sure the app is installed + local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id)) + if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]] + then + ynh_print_err --message="$app is not in the apps list" + exit 1 + fi - # Make sure the app has its own user - if ! id -u "$app" &>/dev/null; then - ynh_print_err --message="There is no \"$app\" system user" - exit 1 - fi + # Make sure the app has its own user + if ! id -u "$app" &>/dev/null; then + ynh_print_err --message="There is no \"$app\" system user" + exit 1 + fi - # Make sure the app has an install_dir setting - local install_dir=$(ynh_app_setting_get --key=install_dir) - if [ -z "$install_dir" ] - then - ynh_print_err --message="$app has no install_dir setting (does it use packaging format >=2?)" - exit 1 - fi + # Make sure the app has an install_dir setting + local install_dir=$(ynh_app_setting_get --key=install_dir) + if [ -z "$install_dir" ] + then + ynh_print_err --message="$app has no install_dir setting (does it use packaging format >=2?)" + exit 1 + fi - # Load the app's service name, or default to $app - local service=$(ynh_app_setting_get --key=service) - [ -z "$service" ] && service=$app; + # Load the app's service name, or default to $app + local service=$(ynh_app_setting_get --key=service) + [ -z "$service" ] && service=$app; - # Export HOME variable - export HOME=$install_dir; + # Export HOME variable + export HOME=$install_dir; - # Load the Environment variables from the app's service - local env_var=$(systemctl show $service.service -p "Environment" --value) - [ -n "$env_var" ] && export $env_var; + # Load the Environment variables from the app's service + local env_var=$(systemctl show $service.service -p "Environment" --value) + [ -n "$env_var" ] && export $env_var; - # Force `php` to its intended version - # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` - local phpversion=$(ynh_app_setting_get --key=phpversion) - local phpflags=$(ynh_app_setting_get --key=phpflags) - if [ -n "$phpversion" ] - then - eval "php() { php${phpversion} ${phpflags} \"\$@\"; }" - export -f php - fi + # Force `php` to its intended version + # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` + local phpversion=$(ynh_app_setting_get --key=phpversion) + local phpflags=$(ynh_app_setting_get --key=phpflags) + if [ -n "$phpversion" ] + then + eval "php() { php${phpversion} ${phpflags} \"\$@\"; }" + export -f php + fi - # Source the EnvironmentFiles from the app's service - local env_files=($(systemctl show $service.service -p "EnvironmentFiles" --value)) - if [ ${#env_files[*]} -gt 0 ] - then - # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. - set -a - for file in ${env_files[*]} - do - [[ $file = /* ]] && source $file - done - set +a - fi + # Source the EnvironmentFiles from the app's service + local env_files=($(systemctl show $service.service -p "EnvironmentFiles" --value)) + if [ ${#env_files[*]} -gt 0 ] + then + # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. + set -a + for file in ${env_files[*]} + do + [[ $file = /* ]] && source $file + done + set +a + fi - # Activate the Python environment, if it exists - if [ -f $install_dir/venv/bin/activate ] - then - # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. - set -a - source $install_dir/venv/bin/activate - set +a - fi + # Activate the Python environment, if it exists + if [ -f $install_dir/venv/bin/activate ] + then + # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. + set -a + source $install_dir/venv/bin/activate + set +a + fi - # cd into the WorkingDirectory set in the service, or default to the install_dir - local env_dir=$(systemctl show $service.service -p "WorkingDirectory" --value) - [ -z $env_dir ] && env_dir=$install_dir; - cd $env_dir + # cd into the WorkingDirectory set in the service, or default to the install_dir + local env_dir=$(systemctl show $service.service -p "WorkingDirectory" --value) + [ -z $env_dir ] && env_dir=$install_dir; + cd $env_dir - # Spawn the app shell - su -s /bin/bash $app + # Spawn the app shell + su -s /bin/bash $app } From 1b03058858fc01c470057fee52f0d22abfcf0be5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 21:47:32 +0200 Subject: [PATCH 011/146] helpers 2.1: simplify the apt extra repo clusterfuck / unused args etc --- helpers/helpers.v2.1.d/apt | 187 ++++++------------------------------- 1 file changed, 27 insertions(+), 160 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 2553cdc03..1309042f8 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -339,13 +339,13 @@ EOF # # [packagingv1] # -# usage: ynh_add_app_dependencies --package=phpversion [--replace] +# usage: ynh_add_app_dependencies --package=phpversion # | arg: -p, --package= - Packages to add as dependencies for the app. # # Requires YunoHost version 3.8.1 or higher. ynh_add_app_dependencies() { # ============ Argument parsing ============= - local -A args_array=([p]=package= [r]=replace) + local -A args_array=([p]=package=) local package ynh_handle_getopts_args "$@" # =========================================== @@ -392,31 +392,23 @@ ynh_remove_app_dependencies() { # # [packagingv1] # -# usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" [--key=key_url] [--name=name] +# usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" --key=key_url # | arg: -r, --repo= - Complete url of the extra repository. # | arg: -p, --package= - The packages to install from this extra repository # | arg: -k, --key= - url to get the public key. -# | arg: -n, --name= - Name for the files for this repo, $app as default value. # # Requires YunoHost version 3.8.1 or higher. ynh_install_extra_app_dependencies() { # ============ Argument parsing ============= - local -A args_array=([r]=repo= [p]=package= [k]=key= [n]=name=) + local -A args_array=([r]=repo= [p]=package= [k]=key=) local repo local package local key - local name ynh_handle_getopts_args "$@" - name="${name:-$app}" - key=${key:-} # =========================================== - # Set a key only if asked - if [ -n "$key" ]; then - key="--key=$key" - fi # Add an extra repository for those packages - ynh_install_extra_repo --repo="$repo" $key --priority=995 --name=$name + ynh_install_extra_repo --repo="$repo" --key=$key # Install requested dependencies from this extra repository. ynh_install_app_dependencies "$package" @@ -428,76 +420,53 @@ ynh_install_extra_app_dependencies() { [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed # Remove this extra repository after packages are installed - ynh_remove_extra_repo --name=$name + ynh_remove_extra_repo } # Add an extra repository correctly, pin it and get the key. # # [internal] # -# usage: ynh_install_extra_repo --repo="repo" [--key=key_url] [--priority=priority_value] [--name=name] [--append] +# usage: ynh_install_extra_repo --repo="repo" [--key=key_url] # | arg: -r, --repo= - Complete url of the extra repository. # | arg: -k, --key= - url to get the public key. -# | arg: -p, --priority= - Priority for the pin -# | arg: -n, --name= - Name for the files for this repo, $app as default value. -# | arg: -a, --append - Do not overwrite existing files. # # Requires YunoHost version 3.8.1 or higher. ynh_install_extra_repo() { # ============ Argument parsing ============= - local -A args_array=([r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append) + local -A args_array=([r]=repo= [k]=key=) local repo local key - local priority - local name - local append ynh_handle_getopts_args "$@" - name="${name:-$app}" - append=${append:-0} - key=${key:-} - priority=${priority:-} # =========================================== - if [ $append -eq 1 ]; then - append="--append" - wget_append="tee --append" - else - append="" - wget_append="tee" - fi - # Split the repository into uri, suite and components. - # Remove "deb " at the beginning of the repo. repo="${repo#deb }" - - # Get the uri local uri="$(echo "$repo" | awk '{ print $1 }')" - - # Get the suite local suite="$(echo "$repo" | awk '{ print $2 }')" - - # Get the components local component="${repo##$uri $suite }" - # Add the repository into sources.list.d - ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append + # Add the new repo in sources.list.d + mkdir --parents "/etc/apt/sources.list.d" + echo "deb $uri $suite $component" > "/etc/apt/sources.list.d/$app.list" # Pin the new repo with the default priority, so it won't be used for upgrades. # Build $pin from the uri without http and any sub path local pin="${uri#*://}" pin="${pin%%/*}" - # Set a priority only if asked - if [ -n "$priority" ]; then - priority="--priority=$priority" - fi - ynh_pin_repo --package="*" --pin="origin \"$pin\"" $priority --name="$name" $append + + # Pin repository + mkdir --parents "/etc/apt/preferences.d" + cat << EOF > "/etc/apt/preferences.d/$app" +Package: * +Pin: origin $pin +Pin-Priority: 995 +EOF # Get the public key for the repo - if [ -n "$key" ]; then - mkdir --parents "/etc/apt/trusted.gpg.d" - # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) - wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg >/dev/null - fi + mkdir --parents "/etc/apt/trusted.gpg.d" + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | tee /etc/apt/trusted.gpg.d/$name.gpg >/dev/null # Update the list of package with the new repo ynh_package_update @@ -507,117 +476,15 @@ ynh_install_extra_repo() { # # [internal] # -# usage: ynh_remove_extra_repo [--name=name] -# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# usage: ynh_remove_extra_repo # # Requires YunoHost version 3.8.1 or higher. ynh_remove_extra_repo() { - # ============ Argument parsing ============= - local -A args_array=([n]=name=) - local name - ynh_handle_getopts_args "$@" - name="${name:-$app}" - # =========================================== - ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list" - # Sury pinning is managed by the regenconf in the core... - [[ "$name" == "extra_php_version" ]] || ynh_secure_remove --file="/etc/apt/preferences.d/$name" - if [ -e /etc/apt/trusted.gpg.d/$name.gpg ]; then - ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg" + ynh_secure_remove --file="/etc/apt/sources.list.d/$app.list" + ynh_secure_remove --file="/etc/apt/preferences.d/$app" + if [ -e /etc/apt/trusted.gpg.d/$app.gpg ]; then + ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$app.gpg" fi - - # (Do we even create a .asc file anywhere ...?) - if [ -e /etc/apt/trusted.gpg.d/$name.asc ]; then - ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.asc" - fi - - # Update the list of package to exclude the old repo ynh_package_update } - -# Add a repository. -# -# [internal] -# -# usage: ynh_add_repo --uri=uri --suite=suite --component=component [--name=name] [--append] -# | arg: -u, --uri= - Uri of the repository. -# | arg: -s, --suite= - Suite of the repository. -# | arg: -c, --component= - Component of the repository. -# | arg: -n, --name= - Name for the files for this repo, $app as default value. -# | arg: -a, --append - Do not overwrite existing files. -# -# Example for a repo like deb http://forge.yunohost.org/debian/ stretch stable -# uri suite component -# ynh_add_repo --uri=http://forge.yunohost.org/debian/ --suite=stretch --component=stable -# -# Requires YunoHost version 3.8.1 or higher. -ynh_add_repo() { - # ============ Argument parsing ============= - local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append) - local uri - local suite - local component - local name - local append - ynh_handle_getopts_args "$@" - name="${name:-$app}" - append=${append:-0} - # =========================================== - - if [ $append -eq 1 ]; then - append="tee --append" - else - append="tee" - fi - - mkdir --parents "/etc/apt/sources.list.d" - # Add the new repo in sources.list.d - echo "deb $uri $suite $component" \ - | $append "/etc/apt/sources.list.d/$name.list" -} - -# Pin a repository. -# -# [internal] -# -# usage: ynh_pin_repo --package=packages --pin=pin_filter [--priority=priority_value] [--name=name] [--append] -# | arg: -p, --package= - Packages concerned by the pin. Or all, *. -# | arg: -i, --pin= - Filter for the pin. -# | arg: -p, --priority= - Priority for the pin -# | arg: -n, --name= - Name for the files for this repo, $app as default value. -# | arg: -a, --append - Do not overwrite existing files. -# -# See https://manpages.debian.org/stretch/apt/apt_preferences.5.en.html#How_APT_Interprets_Priorities for information about pinning. -# -# Requires YunoHost version 3.8.1 or higher. -ynh_pin_repo() { - # ============ Argument parsing ============= - local -A args_array=([p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append) - local package - local pin - local priority - local name - local append - ynh_handle_getopts_args "$@" - package="${package:-*}" - priority=${priority:-50} - name="${name:-$app}" - append=${append:-0} - # =========================================== - - if [ $append -eq 1 ]; then - append="tee --append" - else - append="tee" - fi - - # Sury pinning is managed by the regenconf in the core... - [[ "$name" != "extra_php_version" ]] || return 0 - - mkdir --parents "/etc/apt/preferences.d" - echo "Package: $package -Pin: $pin -Pin-Priority: $priority -" \ - | $append "/etc/apt/preferences.d/$name" -} From 94c12c6409577538c3560b1feb325a7791b59507 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 22:38:53 +0200 Subject: [PATCH 012/146] helpers 2.1: heavily simplify php-related helpers: remove --phpversion argument, --composerversion (define YNH_COMPOSER_VERSION prior to calling helper), --usage/--footprint (can still be customized by defining fpm_usage/footprint setting) --- helpers/helpers.v2.1.d/php | 175 ++++++++----------------------------- 1 file changed, 37 insertions(+), 138 deletions(-) diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index d32e69392..7371f585e 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -9,7 +9,7 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # # usage: ynh_add_fpm_config # -# Case 1 (recommended) : your provided a snippet conf/extra_php-fpm.conf +# This helper assumes the app has an conf/extra_php-fpm.conf snippet # # The actual PHP configuration will be automatically generated, # and your extra_php-fpm.conf will be appended (typically contains PHP upload limits) @@ -26,25 +26,6 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # Otherwise, if you want the user to have control over these, we encourage to create a config panel # (which should ultimately be standardized by the core ...) # -# Case 2 (deprecate) : you provided an entire conf/php-fpm.conf -# -# The configuration will be hydrated, replacing __FOOBAR__ placeholders with $foobar values, etc. -# -# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf -# -# ---------------------- -# -# fpm_footprint: Memory footprint of the service (low/medium/high). -# low - Less than 20 MB of RAM by pool. -# medium - Between 20 MB and 40 MB of RAM by pool. -# high - More than 40 MB of RAM by pool. -# N - Or you can specify a quantitative footprint as MB by pool (use watch -n0.5 ps -o user,cmd,%cpu,rss -u APP) -# -# fpm_usage: Expected usage of the service (low/medium/high). -# low - Personal usage, behind the SSO. -# medium - Low usage, few people or/and publicly accessible. -# high - High usage, frequently visited website. -# # The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM. # So it will be used to defined 'pm.max_children' # A lower value for the footprint will allow more children for 'pm.max_children'. And so for @@ -69,52 +50,15 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # Requires YunoHost version 4.1.0 or higher. ynh_add_fpm_config() { # ============ Argument parsing ============= - local _globalphpversion=${phpversion-:} - local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [g]=group=) + local -A args_array=([g]=group=) local group - local phpversion - local usage - local footprint ynh_handle_getopts_args "$@" group=${group:-} # =========================================== - # The default behaviour is to use the template. - local autogenconf=false - usage="${usage:-}" - footprint="${footprint:-}" - if [ -n "$usage" ] || [ -n "$footprint" ] || [[ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]]; then - autogenconf=true - - # If no usage provided, default to the value existing in setting ... or to low - local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) - if [ -z "$usage" ] - then - usage=${fpm_usage_in_setting:-low} - ynh_app_setting_set --key=fpm_usage --value=$usage - fi - - # If no footprint provided, default to the value existing in setting ... or to low - local fpm_footprint_in_setting=$(ynh_app_setting_get --key=fpm_footprint) - if [ -z "$footprint" ] - then - footprint=${fpm_footprint_in_setting:-low} - ynh_app_setting_set --key=fpm_footprint --value=$footprint - fi - - fi - - # Set the default PHP-FPM version by default - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - phpversion="${phpversion:-$YNH_PHP_VERSION}" - else - phpversion="${phpversion:-$_globalphpversion}" - fi - - local old_phpversion=$(ynh_app_setting_get --key=phpversion) - # If the PHP version changed, remove the old fpm conf # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper) + local old_phpversion=$(ynh_app_setting_get --key=phpversion) if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" @@ -136,20 +80,12 @@ ynh_add_fpm_config() { ynh_app_setting_set --key=fpm_service --value="$fpm_service" ynh_app_setting_set --key=phpversion --value=$phpversion - if [ $autogenconf == "false" ]; then - # Usage 1, use the template in conf/php-fpm.conf - local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" - # Make sure now that the template indeed exists - [ -e "$phpfpm_path" ] || ynh_die --message="Unable to find template to configure PHP-FPM." - else - # Usage 2, generate a PHP-FPM config file with ynh_get_scalable_phpfpm + # Define the values to use for the configuration of PHP. + ynh_get_scalable_phpfpm - # Define the values to use for the configuration of PHP. - ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint - - local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app") - local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" - echo " + local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app") + local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" + echo " [__APP__] user = __APP__ @@ -167,23 +103,22 @@ pm.max_requests = 500 request_terminate_timeout = 1d " >"$phpfpm_path" - if [ "$php_pm" = "dynamic" ]; then - echo " + if [ "$php_pm" = "dynamic" ]; then + echo " pm.start_servers = __PHP_START_SERVERS__ pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__ pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__ " >>"$phpfpm_path" - elif [ "$php_pm" = "ondemand" ]; then - echo " + elif [ "$php_pm" = "ondemand" ]; then + echo " pm.process_idle_timeout = 10s " >>"$phpfpm_path" - fi + fi - # Concatene the extra config. - if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then - cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path" - fi + # Concatene the extra config. + if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then + cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path" fi local finalphpconf="$fpm_config_dir/pool.d/$app.conf" @@ -206,17 +141,6 @@ pm.process_idle_timeout = 10s ynh_remove_fpm_config() { local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local fpm_service=$(ynh_app_setting_get --key=fpm_service) - # Get the version of PHP used by this app - local phpversion=$(ynh_app_setting_get --key=phpversion) - - # Assume default PHP-FPM version by default - phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}" - - # Assume default PHP files if not set - if [ -z "$fpm_config_dir" ]; then - fpm_config_dir="/etc/php/$YNH_DEFAULT_PHP_VERSION/fpm" - fpm_service="php$YNH_DEFAULT_PHP_VERSION-fpm" - fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" ynh_systemd_action --service_name=$fpm_service --action=reload @@ -226,8 +150,8 @@ ynh_remove_fpm_config() { # # [internal] # -# usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print] -# | arg: -f, --footprint= - Memory footprint of the service (low/medium/high). +# usage: ynh_get_scalable_phpfpm +# Footprint can be defined via the "fpm_footprint", to be set prior to calling this helper # low - Less than 20 MB of RAM by pool. # medium - Between 20 MB and 40 MB of RAM by pool. # high - More than 40 MB of RAM by pool. @@ -235,24 +159,24 @@ ynh_remove_fpm_config() { # To have this value, use the following command and stress the service. # watch -n0.5 ps -o user,cmd,%cpu,rss -u APP # -# | arg: -u, --usage= - Expected usage of the service (low/medium/high). +# Usage can be defined via the "fpm_usage", to be set prior to calling this helper # low - Personal usage, behind the SSO. # medium - Low usage, few people or/and publicly accessible. # high - High usage, frequently visited website. # -# | arg: -p, --print - Print the result (intended for debug purpose only when packaging the app) ynh_get_scalable_phpfpm() { - # ============ Argument parsing ============= - local -A args_array=([u]=usage= [f]=footprint= [p]=print) - local usage - local footprint - local print - ynh_handle_getopts_args "$@" - # Set all characters as lowercase - footprint=${footprint,,} - usage=${usage,,} - print=${print:-0} + # If no usage provided, default to the value existing in setting ... or to low + local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) + local usage=${fpm_usage_in_setting:-low} + ynh_app_setting_set --key=fpm_usage --value=$usage + + # If no footprint provided, default to the value existing in setting ... or to low + local fpm_footprint_in_setting=$(ynh_app_setting_get --key=fpm_footprint) + local footprint=${fpm_footprint_in_setting:-low} + ynh_app_setting_set --key=fpm_footprint --value=$footprint + + # Set all characters as lowercase if [ "$footprint" = "low" ]; then footprint=20 elif [ "$footprint" = "medium" ]; then @@ -364,27 +288,19 @@ YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} # Execute a command with Composer # -# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$install_dir] --commands="commands" -# | arg: -v, --phpversion - PHP version to use with composer +# usage: ynh_composer_exec [--workdir=$install_dir] --commands="commands" # | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path # | arg: -c, --commands - Commands to execute. # # Requires YunoHost version 4.2 or higher. ynh_composer_exec() { # ============ Argument parsing ============= - local _globalphpversion=${phpversion-:} - local -A args_array=([v]=phpversion= [w]=workdir= [c]=commands=) - local phpversion + local -A args_array=([w]=workdir= [c]=commands=) local workdir local commands ynh_handle_getopts_args "$@" workdir="${workdir:-${install_dir:-$final_path}}" - - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - phpversion="${phpversion:-$YNH_PHP_VERSION}" - else - phpversion="${phpversion:-$_globalphpversion}" - fi + # =========================================== COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ php${phpversion} "$workdir/composer.phar" $commands \ @@ -393,42 +309,25 @@ ynh_composer_exec() { # Install and initialize Composer in the given directory # -# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$install_dir] [--install_args="--optimize-autoloader"] [--composerversion=composerversion] -# | arg: -v, --phpversion - PHP version to use with composer +# usage: ynh_install_composer [--workdir=$install_dir] [--install_args="--optimize-autoloader"] # | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. # | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include -# | arg: -c, --composerversion - Composer version to install # # Requires YunoHost version 4.2 or higher. ynh_install_composer() { # ============ Argument parsing ============= - local _globalphpversion=${phpversion-:} - local -A args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) - local phpversion + local -A args_array=([w]=workdir= [a]=install_args=) local workdir local install_args - local composerversion ynh_handle_getopts_args "$@" # =========================================== - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - workdir="${workdir:-$final_path}" - else - workdir="${workdir:-$install_dir}" - fi - - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - phpversion="${phpversion:-$YNH_PHP_VERSION}" - else - phpversion="${phpversion:-$_globalphpversion}" - fi - + workdir="${workdir:-$install_dir}" install_args="${install_args:-}" - composerversion="${composerversion:-$YNH_COMPOSER_VERSION}" curl -sS https://getcomposer.org/installer \ | COMPOSER_HOME="$workdir/.composer" \ - php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \ + php${phpversion} -- --quiet --install-dir="$workdir" --version=$YNH_COMPOSER_VERSION \ || ynh_die --message="Unable to install Composer." # install dependencies From 6b577bdd23e7cba2b05049bc66c365ffe905e80f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 22:39:43 +0200 Subject: [PATCH 013/146] helpers 2.1: remove packaging v1 ynh_backup_before_upgrade and ynh_restore_upgradebackup, handled by core in packaging v2 --- helpers/helpers.v2.1.d/backup | 91 ----------------------------------- 1 file changed, 91 deletions(-) diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 820a2562d..bcf322914 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -408,94 +408,3 @@ ynh_backup_archive_exists() { yunohost backup list --output-as json --quiet \ | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null } - -# Make a backup in case of failed upgrade -# -# [packagingv1] -# -# usage: ynh_backup_before_upgrade -# -# Usage in a package script: -# ``` -# ynh_backup_before_upgrade -# ynh_clean_setup () { -# ynh_restore_upgradebackup -# } -# ynh_abort_if_errors -# ``` -# -# Requires YunoHost version 2.7.2 or higher. -ynh_backup_before_upgrade() { - if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]; then - ynh_print_warn --message="This app doesn't have any backup script." - return - fi - backup_number=1 - local old_backup_number=2 - local app_bck=${app//_/-} # Replace all '_' by '-' - NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} - - if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then - # Check if a backup already exists with the prefix 1 - if ynh_backup_archive_exists "$app_bck-pre-upgrade1"; then - # Prefix becomes 2 to preserve the previous backup - backup_number=2 - old_backup_number=1 - fi - - # Create backup - BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug - if [ "$?" -eq 0 ]; then - # If the backup succeeded, remove the previous backup - if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then - # Remove the previous backup only if it exists - yunohost backup delete $app_bck-pre-upgrade$old_backup_number >/dev/null - fi - else - ynh_die --message="Backup failed, the upgrade process was aborted." - fi - else - ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" - fi -} - -# Restore a previous backup if the upgrade process failed -# -# [packagingv1] -# -# usage: ynh_restore_upgradebackup -# -# Usage in a package script: -# ``` -# ynh_backup_before_upgrade -# ynh_clean_setup () { -# ynh_restore_upgradebackup -# } -# ynh_abort_if_errors -# ``` -# -# Requires YunoHost version 2.7.2 or higher. -ynh_restore_upgradebackup() { - ynh_print_err --message="Upgrade failed." - local app_bck=${app//_/-} # Replace all '_' by '-' - - NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} - - if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then - # Check if an existing backup can be found before removing and restoring the application. - if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"; then - # Remove the application then restore it - yunohost app remove $app - # Restore the backup - yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug - if [[ -d /etc/yunohost/apps/$app ]] - then - ynh_die --message="The app was restored to the way it was before the failed upgrade." - else - ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|" - fi - fi - else - ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" - fi -} From b79ff15d32413b66f56f50e1a12584de17de0cbc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 22:40:35 +0200 Subject: [PATCH 014/146] helpers 2.1: remove legacy --nonappend/--non-append arg in logrotate helper --- helpers/helpers.v2.1.d/logrotate | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index 970377acf..84f305195 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -14,18 +14,6 @@ FIRST_CALL_TO_LOGROTATE="true" # Requires YunoHost version 2.6.4 or higher. ynh_use_logrotate() { - # Stupid patch to ignore legacy --non-append and --nonappend - # which was never properly understood and improperly used and kind of bullshit - local all_args=( ${@} ) - for I in $(seq 0 $(($# - 1))) - do - if [[ "${all_args[$I]}" == "--non-append" ]] || [[ "${all_args[$I]}" == "--nonappend" ]] - then - unset all_args[$I] - fi - done - set -- "${all_args[@]}" - # ============ Argument parsing ============= local -A args_array=([l]=logfile= [u]=specific_user=) local logfile From 8b59e315bf2a3503b9a71befcad1244fb790c880 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 22:42:52 +0200 Subject: [PATCH 015/146] helpers 2.1: remove legacy ynh_psql_test_if_first_run --- helpers/helpers.v2.1.d/postgresql | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index b28e11e38..5423026d2 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -282,21 +282,3 @@ ynh_psql_remove_db() { ynh_print_warn --message="User $db_user not found" fi } - -# Create a master password and set up global settings -# -# [internal] -# -# usage: ynh_psql_test_if_first_run -# -# It also make sure that postgresql is installed and running -# Please always call this script in install and restore scripts -# -# Requires YunoHost version 2.7.13 or higher. -ynh_psql_test_if_first_run() { - - # Make sure postgresql is indeed installed - dpkg --list | grep -q "ii postgresql-$PSQL_VERSION" || ynh_die --message="postgresql-$PSQL_VERSION is not installed !?" - - yunohost tools regen-conf postgresql -} From f895724a25046044127ba66127adca71148405d1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 22:45:25 +0200 Subject: [PATCH 016/146] helpers 2.1: remove legacy ynh_webpath_available and ynh_webpath_register --- helpers/helpers.v2.1.d/setting | 46 ---------------------------------- 1 file changed, 46 deletions(-) diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 67c4d5f0d..60dbf0efc 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -95,49 +95,3 @@ else: EOF set -o xtrace # set -x } - -# Check availability of a web path -# -# [packagingv1] -# -# usage: ynh_webpath_available --domain=domain --path_url=path -# | arg: -d, --domain= - the domain/host of the url -# | arg: -p, --path_url= - the web path to check the availability of -# -# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee -# -# Requires YunoHost version 2.6.4 or higher. -ynh_webpath_available() { - # ============ Argument parsing ============= - local -A args_array=([d]=domain= [p]=path_url=) - local domain - local path_url - ynh_handle_getopts_args "$@" - # =========================================== - - yunohost domain url-available $domain $path_url -} - -# Register/book a web path for an app -# -# [packagingv1] -# -# usage: ynh_webpath_register --app=app --domain=domain --path_url=path -# | arg: -a, --app= - the app for which the domain should be registered -# | arg: -d, --domain= - the domain/host of the web path -# | arg: -p, --path_url= - the web path to be registered -# -# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee -# -# Requires YunoHost version 2.6.4 or higher. -ynh_webpath_register() { - # ============ Argument parsing ============= - local -A args_array=([a]=app= [d]=domain= [p]=path_url=) - local app - local domain - local path_url - ynh_handle_getopts_args "$@" - # =========================================== - - yunohost app register-url $app $domain $path_url -} From a240fb231695b3ee79c864dc533b2e262b9f4e94 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 22:50:00 +0200 Subject: [PATCH 017/146] helpers 2.1: use positional args for ynh_normalize_url_path --- helpers/helpers.v2.1.d/string | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/helpers/helpers.v2.1.d/string b/helpers/helpers.v2.1.d/string index 398facec2..7990e2b19 100644 --- a/helpers/helpers.v2.1.d/string +++ b/helpers/helpers.v2.1.d/string @@ -112,8 +112,6 @@ ynh_sanitize_dbid() { # Normalize the url path syntax # -# [internal] -# # Handle the slash at the beginning of path and its absence at ending # Return a normalized url path # @@ -124,16 +122,11 @@ ynh_sanitize_dbid() { # ynh_normalize_url_path /example/ # -> /example # ynh_normalize_url_path / # -> / # -# usage: ynh_normalize_url_path --path_url=path_to_normalize -# | arg: -p, --path_url= - URL path to normalize before using it +# usage: ynh_normalize_url_path path_to_normalize # # Requires YunoHost version 2.6.4 or higher. ynh_normalize_url_path() { - # ============ Argument parsing ============= - local -A args_array=([p]=path_url=) - local path_url - ynh_handle_getopts_args "$@" - # =========================================== + local path_url=$1 test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / From 3a500d8457adb54a62bc657501e39ea77b9c53e4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 23:38:22 +0200 Subject: [PATCH 018/146] helpers 2.1: remove ynh_require_ram and unused --ignore_swap and --only_swap options in ynh_get_ram --- helpers/helpers.v2.1.d/hardware | 103 -------------------------------- helpers/helpers.v2.1.d/utils | 38 ++++++++++++ 2 files changed, 38 insertions(+), 103 deletions(-) delete mode 100644 helpers/helpers.v2.1.d/hardware diff --git a/helpers/helpers.v2.1.d/hardware b/helpers/helpers.v2.1.d/hardware deleted file mode 100644 index 678f381b1..000000000 --- a/helpers/helpers.v2.1.d/hardware +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# Get the total or free amount of RAM+swap on the system -# -# [packagingv1] -# -# usage: ynh_get_ram [--free|--total] [--ignore_swap|--only_swap] -# | arg: -f, --free - Count free RAM+swap -# | arg: -t, --total - Count total RAM+swap -# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM -# | arg: -o, --only_swap - Ignore real RAM, consider only swap -# | ret: the amount of free ram, in MB (MegaBytes) -# -# Requires YunoHost version 3.8.1 or higher. -ynh_get_ram() { - # ============ Argument parsing ============= - local -A args_array=([f]=free [t]=total [s]=ignore_swap [o]=only_swap) - local free - local total - local ignore_swap - local only_swap - ynh_handle_getopts_args "$@" - ignore_swap=${ignore_swap:-0} - only_swap=${only_swap:-0} - free=${free:-0} - total=${total:-0} - # =========================================== - - if [ $free -eq $total ]; then - ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram" - ram=0 - # Use the total amount of ram - elif [ $free -eq 1 ]; then - local free_ram=$(LC_ALL=C vmstat --stats --unit M | grep "free memory" | awk '{print $1}') - local free_swap=$(LC_ALL=C vmstat --stats --unit M | grep "free swap" | awk '{print $1}') - local free_ram_swap=$((free_ram + free_swap)) - - # Use the total amount of free ram - local ram=$free_ram_swap - if [ $ignore_swap -eq 1 ]; then - # Use only the amount of free ram - ram=$free_ram - elif [ $only_swap -eq 1 ]; then - # Use only the amount of free swap - ram=$free_swap - fi - elif [ $total -eq 1 ]; then - local total_ram=$(LC_ALL=C vmstat --stats --unit M | grep "total memory" | awk '{print $1}') - local total_swap=$(LC_ALL=C vmstat --stats --unit M | grep "total swap" | awk '{print $1}') - local total_ram_swap=$((total_ram + total_swap)) - - local ram=$total_ram_swap - if [ $ignore_swap -eq 1 ]; then - # Use only the amount of free ram - ram=$total_ram - elif [ $only_swap -eq 1 ]; then - # Use only the amount of free swap - ram=$total_swap - fi - fi - - echo $ram -} - -# Return 0 or 1 depending if the system has a given amount of RAM+swap free or total -# -# [packagingv1] -# -# usage: ynh_require_ram --required=RAM [--free|--total] [--ignore_swap|--only_swap] -# | arg: -r, --required= - The amount to require, in MB -# | arg: -f, --free - Count free RAM+swap -# | arg: -t, --total - Count total RAM+swap -# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM -# | arg: -o, --only_swap - Ignore real RAM, consider only swap -# | ret: 1 if the ram is under the requirement, 0 otherwise. -# -# Requires YunoHost version 3.8.1 or higher. -ynh_require_ram() { - # ============ Argument parsing ============= - local -A args_array=([r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap) - local required - local free - local total - local ignore_swap - local only_swap - ynh_handle_getopts_args "$@" - # Dunno if that's the right way to do, but that's some black magic to be able to - # forward the bool args to ynh_get_ram easily? - # If the variable $free is not empty, set it to '--free' - free=${free:+--free} - total=${total:+--total} - ignore_swap=${ignore_swap:+--ignore_swap} - only_swap=${only_swap:+--only_swap} - # =========================================== - - local ram=$(ynh_get_ram $free $total $ignore_swap $only_swap) - - if [ $ram -lt $required ]; then - return 1 - else - return 0 - fi -} diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 5c3e2b2f6..f82a7c265 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -1068,3 +1068,41 @@ except socket.error: sys.exit(0) EOF } + +# Get the total or free amount of RAM+swap on the system +# +# [packagingv1] +# +# usage: ynh_get_ram [--free|--total] +# | arg: -f, --free - Count free RAM+swap +# | arg: -t, --total - Count total RAM+swap +# | ret: the amount of free ram, in MB (MegaBytes) +# +# Requires YunoHost version 3.8.1 or higher. +ynh_get_ram() { + # ============ Argument parsing ============= + local -A args_array=([f]=free [t]=total) + local free + local total + ynh_handle_getopts_args "$@" + free=${free:-0} + total=${total:-0} + # =========================================== + + if [ $free -eq $total ]; then + ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram" + ram=0 + elif [ $free -eq 1 ]; then + local free_ram=$(LC_ALL=C vmstat --stats --unit M | grep "free memory" | awk '{print $1}') + local free_swap=$(LC_ALL=C vmstat --stats --unit M | grep "free swap" | awk '{print $1}') + local free_ram_swap=$((free_ram + free_swap)) + local ram=$free_ram_swap + elif [ $total -eq 1 ]; then + local total_ram=$(LC_ALL=C vmstat --stats --unit M | grep "total memory" | awk '{print $1}') + local total_swap=$(LC_ALL=C vmstat --stats --unit M | grep "total swap" | awk '{print $1}') + local total_ram_swap=$((total_ram + total_swap)) + local ram=$total_ram_swap + fi + + echo $ram +} From 70f51541303dcc839aac6f55acf72d80b54a8597 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 23:40:44 +0200 Subject: [PATCH 019/146] helpers 2.1: remove ynh_get_debian_release, apps should just use $YNH_DEBIAN_VERSION --- helpers/helpers.v2.1.d/utils | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index f82a7c265..706cb719c 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -775,18 +775,6 @@ ynh_render_template() { ).render(os.environ));' <$template_path >$output_path } -# Fetch the Debian release codename -# -# [packagingv1] -# -# usage: ynh_get_debian_release -# | ret: The Debian release codename (i.e. jessie, stretch, ...) -# -# Requires YunoHost version 2.7.12 or higher. -ynh_get_debian_release() { - echo $(lsb_release --codename --short) -} - _acceptable_path_to_delete() { local file=$1 From 84a7b23e8a3831c8b719844376b9277999300420 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 23:42:53 +0200 Subject: [PATCH 020/146] helpers 2.1: replace ynh_compare_current_package_version with simpler ynh_if_upgrading_from_version_prior_to and ynh_if_upgrading_from_version_prior_or_equal_to --- helpers/helpers.v2.1.d/utils | 54 ++++++++++++------------------------ 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 706cb719c..531b471d4 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -937,48 +937,28 @@ ynh_check_app_version_changed() { echo $return_value } -# Compare the current package version against another version given as an argument. +# Compare the current package version is strictly lower than another version given as an argument # -# usage: ynh_compare_current_package_version --comparison (lt|le|eq|ne|ge|gt) --version -# | arg: --comparison - Comparison type. Could be : `lt` (lower than), `le` (lower or equal), `eq` (equal), `ne` (not equal), `ge` (greater or equal), `gt` (greater than) -# | arg: --version - The version to compare. Need to be a version in the yunohost package version type (like `2.3.1~ynh4`) -# | ret: 0 if the evaluation is true, 1 if false. +# example: if ynh_if_upgrading_from_version_prior_to 2.3.2~ynh1; then ... # -# example: ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1 -# -# This helper is usually used when we need to do some actions only for some old package versions. -# -# Generally you might probably use it as follow in the upgrade script : -# ``` -# if ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1 -# then -# # Do something that is needed for the package version older than 2.3.2~ynh1 -# fi -# ``` -# -# Requires YunoHost version 3.8.0 or higher. -ynh_compare_current_package_version() { - # ============ Argument parsing ============= - local -A args_array=([c]=comparison= [v]=version=) - local version - local comparison - ynh_handle_getopts_args "$@" - # =========================================== +# Requires YunoHost version 11.2 or higher. +ynh_if_upgrading_from_version_prior_to() { + local version=$1 + [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" - local current_version=$YNH_APP_CURRENT_VERSION + dpkg --compare-versions $YNH_APP_CURRENT_VERSION lt $version +} - # Check the syntax of the versions - if [[ ! $version =~ '~ynh' ]] || [[ ! $current_version =~ '~ynh' ]]; then - ynh_die --message="Invalid argument for version." - fi +# Compare the current package version is lower or equal to another version given as an argument +# +# example: if ynh_if_upgrading_from_version_prior_or_equal_to 2.3.2~ynh1; then ... +# +# Requires YunoHost version 11.2 or higher. +ynh_if_upgrading_from_version_prior_or_equal_to() { + local version=$1 + [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" - # Check validity of the comparator - if [[ ! $comparison =~ (lt|le|eq|ne|ge|gt) ]]; then - ynh_die --message="Invalid comparator must be : lt, le, eq, ne, ge, gt" - fi - - # Return the return value of dpkg --compare-versions - dpkg --compare-versions $current_version $comparison $version + dpkg --compare-versions $YNH_APP_CURRENT_VERSION le $version } # Check if we should enforce sane default permissions (= disable rwx for 'others') From 88684c79373d603eb1e6d5db5d10ab7d5da9413d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 23:43:35 +0200 Subject: [PATCH 021/146] helpers 2.1: remove legacy ynh_legacy_permissions_exists and ynh_legacy_permissions_delete_all --- helpers/helpers.v2.1.d/permission | 38 ------------------------------- 1 file changed, 38 deletions(-) diff --git a/helpers/helpers.v2.1.d/permission b/helpers/helpers.v2.1.d/permission index 28847cd08..adf9544ee 100644 --- a/helpers/helpers.v2.1.d/permission +++ b/helpers/helpers.v2.1.d/permission @@ -358,41 +358,3 @@ ynh_permission_has_user() { return 1 } - -# Check if a legacy permissions exist -# -# [packagingv1] -# -# usage: ynh_legacy_permissions_exists -# | exit: Return 1 if the permission doesn't exist, 0 otherwise -# -# Requires YunoHost version 4.1.2 or higher. -ynh_legacy_permissions_exists() { - for permission in "skipped" "unprotected" "protected"; do - if ynh_permission_exists --permission="legacy_${permission}_uris"; then - return 0 - fi - done - return 1 -} - -# Remove all legacy permissions -# -# [packagingv1] -# -# usage: ynh_legacy_permissions_delete_all -# -# example: -# if ynh_legacy_permissions_exists -# then -# ynh_legacy_permissions_delete_all -# # You can recreate the required permissions here with ynh_permission_create -# fi -# Requires YunoHost version 4.1.2 or higher. -ynh_legacy_permissions_delete_all() { - for permission in "skipped" "unprotected" "protected"; do - if ynh_permission_exists --permission="legacy_${permission}_uris"; then - ynh_permission_delete --permission="legacy_${permission}_uris" - fi - done -} From 7c1b07ee0fd291bc4907babc182563f09580f868 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 23:45:03 +0200 Subject: [PATCH 022/146] helpers 2.1: Simplify ynh_app_upstream_version and ynh_read_manifest --- helpers/helpers.v2.1.d/utils | 86 ++++-------------------------------- 1 file changed, 9 insertions(+), 77 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 531b471d4..73553983e 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -149,7 +149,7 @@ ynh_setup_source() { source_id="${source_id:-main}" # =========================================== - local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") + local sources_json=$(ynh_read_manifest ".resources.sources[\"$source_id\"]") if jq -re ".url" <<< "$sources_json" then local arch_prefix="" @@ -453,9 +453,9 @@ ynh_add_config() { local template local destination ynh_handle_getopts_args "$@" - local template_path # =========================================== + local template_path if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then template_path="$YNH_APP_BASEDIR/conf/$template" elif [ -f "$template" ]; then @@ -827,95 +827,27 @@ ynh_secure_remove() { set -o xtrace # set -x } -# Read the value of a key in a ynh manifest file +# Read the value of a key in the app's manifest # -# usage: ynh_read_manifest --manifest="manifest.json" --key="key" -# | arg: -m, --manifest= - Path of the manifest to read -# | arg: -k, --key= - Name of the key to find +# usage: ynh_read_manifest "key" +# | arg: key - Name of the key to find # | ret: the value associate to that key # # Requires YunoHost version 3.5.0 or higher. ynh_read_manifest() { - # ============ Argument parsing ============= - local -A args_array=([m]=manifest= [k]=manifest_key=) - local manifest - local manifest_key - ynh_handle_getopts_args "$@" - # =========================================== - - if [ ! -e "${manifest:-}" ]; then - # If the manifest isn't found, try the common place for backup and restore script. - if [ -e "$YNH_APP_BASEDIR/manifest.json" ] - then - manifest="$YNH_APP_BASEDIR/manifest.json" - elif [ -e "$YNH_APP_BASEDIR/manifest.toml" ] - then - manifest="$YNH_APP_BASEDIR/manifest.toml" - else - ynh_die --message="No manifest found !?" - fi - fi - - if echo "$manifest" | grep -q '\.json$' - then - jq ".$manifest_key" "$manifest" --raw-output - else - cat "$manifest" | python3 -c 'import json, toml, sys; print(json.dumps(toml.load(sys.stdin)))' | jq ".$manifest_key" --raw-output - fi + cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".$manifest_key" --raw-output } -# Read the upstream version from the manifest or `$YNH_APP_MANIFEST_VERSION` +# Return the app upstream version, deduced from `$YNH_APP_MANIFEST_VERSION` and strippig the `~ynhX` part # -# usage: ynh_app_upstream_version [--manifest="manifest.json"] -# | arg: -m, --manifest= - Path of the manifest to read +# usage: ynh_app_upstream_version # | ret: the version number of the upstream app # -# If the `manifest` is not specified, the envvar `$YNH_APP_MANIFEST_VERSION` will be used. -# -# The version number in the manifest is defined by `~ynh`. -# # For example, if the manifest contains `4.3-2~ynh3` the function will return `4.3-2` # # Requires YunoHost version 3.5.0 or higher. ynh_app_upstream_version() { - # ============ Argument parsing ============= - local -A args_array=([m]=manifest=) - local manifest - ynh_handle_getopts_args "$@" - manifest="${manifest:-}" - # =========================================== - - if [[ "$manifest" != "" ]] && [[ -e "$manifest" ]]; then - version_key_=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") - else - version_key_=$YNH_APP_MANIFEST_VERSION - fi - - echo "${version_key_/~ynh*/}" -} - -# Read package version from the manifest -# -# [internal] -# -# usage: ynh_app_package_version [--manifest="manifest.json"] -# | arg: -m, --manifest= - Path of the manifest to read -# | ret: the version number of the package -# -# The version number in the manifest is defined by `~ynh`. -# -# For example, if the manifest contains `4.3-2~ynh3` the function will return `3` -# -# Requires YunoHost version 3.5.0 or higher. -ynh_app_package_version() { - # ============ Argument parsing ============= - local -A args_array=([m]=manifest=) - local manifest - ynh_handle_getopts_args "$@" - # =========================================== - - version_key_=$YNH_APP_MANIFEST_VERSION - echo "${version_key_/*~ynh/}" + echo "${$YNH_APP_MANIFEST_VERSION/~ynh*/}" } # Checks the app version to upgrade with the existing app version and returns: From 346f42643b1543632239fc960cc45d95799dad21 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 May 2024 23:48:22 +0200 Subject: [PATCH 023/146] helpers 2.1: move ynh_add_config, ynh_replace_vars, ynh_read_var_in_file, ynh_write_var_in_file and ynh_render_template to a separate 'templating' file --- helpers/helpers.v2.1.d/templating | 367 ++++++++++++++++++++++++++++++ helpers/helpers.v2.1.d/utils | 366 ----------------------------- 2 files changed, 367 insertions(+), 366 deletions(-) create mode 100644 helpers/helpers.v2.1.d/templating diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating new file mode 100644 index 000000000..9ba6f7cff --- /dev/null +++ b/helpers/helpers.v2.1.d/templating @@ -0,0 +1,367 @@ +#!/bin/bash + +# Create a dedicated config file from a template +# +# usage: ynh_add_config --template="template" --destination="destination" +# | arg: -t, --template= - Template config file to use +# | arg: -d, --destination= - Destination of the config file +# +# examples: +# ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" +# ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" +# +# The template can be by default the name of a file in the conf directory +# of a YunoHost Package, a relative path or an absolute path. +# +# The helper will use the template `template` to generate a config file +# `destination` by replacing the following keywords with global variables +# that should be defined before calling this helper : +# ``` +# __PATH__ by $path_url +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# ``` +# And any dynamic variables that should be defined before calling this helper like: +# ``` +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# ``` +# +# The helper will verify the checksum and backup the destination file +# if it's different before applying the new template. +# +# And it will calculate and store the destination file checksum +# into the app settings when configuration is done. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_config() { + # ============ Argument parsing ============= + local -A args_array=([t]=template= [d]=destination=) + local template + local destination + ynh_handle_getopts_args "$@" + # =========================================== + + local template_path + if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then + template_path="$YNH_APP_BASEDIR/conf/$template" + elif [ -f "$template" ]; then + template_path=$template + else + ynh_die --message="The provided template $template doesn't exist" + fi + + ynh_backup_if_checksum_is_different --file="$destination" + + # Make sure to set the permissions before we copy the file + # This is to cover a case where an attacker could have + # created a file beforehand to have control over it + # (cp won't overwrite ownership / modes by default...) + touch $destination + chown root:root $destination + chmod 640 $destination + + cp -f "$template_path" "$destination" + + _ynh_apply_default_permissions $destination + + ynh_replace_vars --file="$destination" + + ynh_store_file_checksum --file="$destination" +} + +# Replace variables in a file +# +# [internal] +# +# usage: ynh_replace_vars --file="file" +# | arg: -f, --file= - File where to replace variables +# +# The helper will replace the following keywords with global variables +# that should be defined before calling this helper : +# __PATH__ by $path_url +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# +# And any dynamic variables that should be defined before calling this helper like: +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# +# Requires YunoHost version 4.1.0 or higher. +ynh_replace_vars() { + # ============ Argument parsing ============= + local -A args_array=([f]=file=) + local file + ynh_handle_getopts_args "$@" + # =========================================== + + # Replace specific YunoHost variables + if test -n "${path_url:-}"; then + # path_url_slash_less is path_url, or a blank value if path_url is only '/' + local path_url_slash_less=${path_url%/} + ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file" + ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" + fi + if test -n "${app:-}"; then + ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" + fi + if test -n "${ynh_node_load_PATH:-}"; then + ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" + fi + + # Replace others variables + + # List other unique (__ __) variables in $file + local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g")) + + set +o xtrace # set +x + + # Do the replacement + local delimit=@ + for one_var in "${uniques_vars[@]}"; do + # Validate that one_var is indeed defined + # -v checks if the variable is defined, for example: + # -v FOO tests if $FOO is defined + # -v $FOO tests if ${!FOO} is defined + # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 + [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" + + # Escape delimiter in match/replace string + match_string="__${one_var^^}__" + match_string=${match_string//${delimit}/"\\${delimit}"} + replace_string="${!one_var}" + replace_string=${replace_string//\\/\\\\} + replace_string=${replace_string//${delimit}/"\\${delimit}"} + + # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs) + sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file" + done + set -o xtrace # set -x +} + +# Get a value from heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_read_var_in_file --file=PATH --key=KEY +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to get +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# This helpers match several var affectation use case in several languages +# We don't use jq or equivalent to keep comments and blank space in files +# This helpers work line by line, it is not able to work correctly +# if you have several identical keys in your files +# +# Example of line this helpers can managed correctly +# .yml +# title: YunoHost documentation +# email: 'yunohost@yunohost.org' +# .json +# "theme": "colib'ris", +# "port": 8102 +# "some_boolean": false, +# "user": null +# .ini +# some_boolean = On +# action = "Clear" +# port = 20 +# .php +# $user= +# user => 20 +# .py +# USER = 8102 +# user = 'https://donate.local' +# CUSTOM['user'] = 'YunoHost' +# +# Requires YunoHost version 4.3 or higher. +ynh_read_var_in_file() { + # ============ Argument parsing ============= + local -A args_array=([f]=file= [k]=key= [a]=after=) + local file + local key + local after + ynh_handle_getopts_args "$@" + after="${after:-}" + # =========================================== + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local line_number=1 + if [[ -n "$after" ]]; then + line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + echo YNH_NULL + return 0 + fi + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + + local first_char="${expression:0:1}" + if [[ "$first_char" == '"' ]]; then + echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + elif [[ "$first_char" == "'" ]]; then + echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + else + echo "$expression" + fi + set -o xtrace # set -x +} + +# Set a value into heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to set +# | arg: -v, --value= - the value to set +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# Requires YunoHost version 4.3 or higher. +ynh_write_var_in_file() { + # ============ Argument parsing ============= + local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) + local file + local key + local value + local after + ynh_handle_getopts_args "$@" + after="${after:-}" + # =========================================== + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local after_line_number=1 + if [[ -n "$after" ]]; then + after_line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$after_line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$after_line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + return 1 + fi + local value_line_number="$(tail +$after_line_number ${file} | grep -m1 -n -i -P $var_part'\K.*$' | cut -d: -f1)" + value_line_number=$((after_line_number + value_line_number)) + local range="${after_line_number},${value_line_number} " + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + endline=${expression_with_comment#"$expression"} + endline="$(echo "$endline" | sed 's/\\/\\\\/g')" + value="$(echo "$value" | sed 's/\\/\\\\/g')" + value=${value//&/"\&"} + local first_char="${expression:0:1}" + delimiter=$'\001' + if [[ "$first_char" == '"' ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # So we need \\\\ to go through 2 sed + value="$(echo "$value" | sed 's/"/\\\\"/g')" + sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file} + elif [[ "$first_char" == "'" ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # However double quotes implies to double \\ to + # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str + value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" + sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file} + else + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]]; then + value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' + fi + if [[ "$ext" =~ ^yaml|yml$ ]]; then + value=" $value" + fi + sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file} + fi + set -o xtrace # set -x +} + +# Render templates with Jinja2 +# +# [internal] +# +# Attention : Variables should be exported before calling this helper to be +# accessible inside templates. +# +# usage: ynh_render_template some_template output_path +# | arg: some_template - Template file to be rendered +# | arg: output_path - The path where the output will be redirected to +ynh_render_template() { + local template_path=$1 + local output_path=$2 + mkdir -p "$(dirname $output_path)" + # Taken from https://stackoverflow.com/a/35009576 + python3 -c 'import os, sys, jinja2; sys.stdout.write( + jinja2.Template(sys.stdin.read() + ).render(os.environ));' <$template_path >$output_path +} diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 73553983e..dadc478c3 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -409,372 +409,6 @@ ynh_local_curl() { fi } -# Create a dedicated config file from a template -# -# usage: ynh_add_config --template="template" --destination="destination" -# | arg: -t, --template= - Template config file to use -# | arg: -d, --destination= - Destination of the config file -# -# examples: -# ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" -# ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" -# -# The template can be by default the name of a file in the conf directory -# of a YunoHost Package, a relative path or an absolute path. -# -# The helper will use the template `template` to generate a config file -# `destination` by replacing the following keywords with global variables -# that should be defined before calling this helper : -# ``` -# __PATH__ by $path_url -# __USER__ by $app -# __FINALPATH__ by $final_path -# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) -# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH -# ``` -# And any dynamic variables that should be defined before calling this helper like: -# ``` -# __DOMAIN__ by $domain -# __APP__ by $app -# __VAR_1__ by $var_1 -# __VAR_2__ by $var_2 -# ``` -# -# The helper will verify the checksum and backup the destination file -# if it's different before applying the new template. -# -# And it will calculate and store the destination file checksum -# into the app settings when configuration is done. -# -# Requires YunoHost version 4.1.0 or higher. -ynh_add_config() { - # ============ Argument parsing ============= - local -A args_array=([t]=template= [d]=destination=) - local template - local destination - ynh_handle_getopts_args "$@" - # =========================================== - - local template_path - if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then - template_path="$YNH_APP_BASEDIR/conf/$template" - elif [ -f "$template" ]; then - template_path=$template - else - ynh_die --message="The provided template $template doesn't exist" - fi - - ynh_backup_if_checksum_is_different --file="$destination" - - # Make sure to set the permissions before we copy the file - # This is to cover a case where an attacker could have - # created a file beforehand to have control over it - # (cp won't overwrite ownership / modes by default...) - touch $destination - chown root:root $destination - chmod 640 $destination - - cp -f "$template_path" "$destination" - - _ynh_apply_default_permissions $destination - - ynh_replace_vars --file="$destination" - - ynh_store_file_checksum --file="$destination" -} - -# Replace variables in a file -# -# [internal] -# -# usage: ynh_replace_vars --file="file" -# | arg: -f, --file= - File where to replace variables -# -# The helper will replace the following keywords with global variables -# that should be defined before calling this helper : -# __PATH__ by $path_url -# __USER__ by $app -# __FINALPATH__ by $final_path -# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) -# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH -# -# And any dynamic variables that should be defined before calling this helper like: -# __DOMAIN__ by $domain -# __APP__ by $app -# __VAR_1__ by $var_1 -# __VAR_2__ by $var_2 -# -# Requires YunoHost version 4.1.0 or higher. -ynh_replace_vars() { - # ============ Argument parsing ============= - local -A args_array=([f]=file=) - local file - ynh_handle_getopts_args "$@" - # =========================================== - - # Replace specific YunoHost variables - if test -n "${path_url:-}"; then - # path_url_slash_less is path_url, or a blank value if path_url is only '/' - local path_url_slash_less=${path_url%/} - ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file" - ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" - fi - if test -n "${app:-}"; then - ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" - fi - if test -n "${ynh_node_load_PATH:-}"; then - ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" - fi - - # Replace others variables - - # List other unique (__ __) variables in $file - local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g")) - - set +o xtrace # set +x - - # Do the replacement - local delimit=@ - for one_var in "${uniques_vars[@]}"; do - # Validate that one_var is indeed defined - # -v checks if the variable is defined, for example: - # -v FOO tests if $FOO is defined - # -v $FOO tests if ${!FOO} is defined - # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 - [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" - - # Escape delimiter in match/replace string - match_string="__${one_var^^}__" - match_string=${match_string//${delimit}/"\\${delimit}"} - replace_string="${!one_var}" - replace_string=${replace_string//\\/\\\\} - replace_string=${replace_string//${delimit}/"\\${delimit}"} - - # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs) - sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file" - done - set -o xtrace # set -x -} - -# Get a value from heterogeneous file (yaml, json, php, python...) -# -# usage: ynh_read_var_in_file --file=PATH --key=KEY -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to get -# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) -# -# This helpers match several var affectation use case in several languages -# We don't use jq or equivalent to keep comments and blank space in files -# This helpers work line by line, it is not able to work correctly -# if you have several identical keys in your files -# -# Example of line this helpers can managed correctly -# .yml -# title: YunoHost documentation -# email: 'yunohost@yunohost.org' -# .json -# "theme": "colib'ris", -# "port": 8102 -# "some_boolean": false, -# "user": null -# .ini -# some_boolean = On -# action = "Clear" -# port = 20 -# .php -# $user= -# user => 20 -# .py -# USER = 8102 -# user = 'https://donate.local' -# CUSTOM['user'] = 'YunoHost' -# -# Requires YunoHost version 4.3 or higher. -ynh_read_var_in_file() { - # ============ Argument parsing ============= - local -A args_array=([f]=file= [k]=key= [a]=after=) - local file - local key - local after - ynh_handle_getopts_args "$@" - after="${after:-}" - # =========================================== - - [[ -f $file ]] || ynh_die --message="File $file does not exists" - - set +o xtrace # set +x - - # Get the line number after which we search for the variable - local line_number=1 - if [[ -n "$after" ]]; then - line_number=$(grep -m1 -n $after $file | cut -d: -f1) - if [[ -z "$line_number" ]]; then - set -o xtrace # set -x - return 1 - fi - fi - - local filename="$(basename -- "$file")" - local ext="${filename##*.}" - local endline=',;' - local assign="=>|:|=" - local comments="#" - local string="\"'" - if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then - endline='#' - fi - if [[ "$ext" =~ ^ini|env$ ]]; then - comments="[;#]" - fi - if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then - comments="//" - fi - local list='\[\s*['$string']?\w+['$string']?\]' - local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' - var_part+="[$string]?${key}[$string]?" - var_part+='\s*\]?\s*' - var_part+="($assign)" - var_part+='\s*' - - # Extract the part after assignation sign - local expression_with_comment="$((tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" - if [[ "$expression_with_comment" == "YNH_NULL" ]]; then - set -o xtrace # set -x - echo YNH_NULL - return 0 - fi - - # Remove comments if needed - local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" - - local first_char="${expression:0:1}" - if [[ "$first_char" == '"' ]]; then - echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' - elif [[ "$first_char" == "'" ]]; then - echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" - else - echo "$expression" - fi - set -o xtrace # set -x -} - -# Set a value into heterogeneous file (yaml, json, php, python...) -# -# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to set -# | arg: -v, --value= - the value to set -# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) -# -# Requires YunoHost version 4.3 or higher. -ynh_write_var_in_file() { - # ============ Argument parsing ============= - local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) - local file - local key - local value - local after - ynh_handle_getopts_args "$@" - after="${after:-}" - # =========================================== - - [[ -f $file ]] || ynh_die --message="File $file does not exists" - - set +o xtrace # set +x - - # Get the line number after which we search for the variable - local after_line_number=1 - if [[ -n "$after" ]]; then - after_line_number=$(grep -m1 -n $after $file | cut -d: -f1) - if [[ -z "$after_line_number" ]]; then - set -o xtrace # set -x - return 1 - fi - fi - - local filename="$(basename -- "$file")" - local ext="${filename##*.}" - local endline=',;' - local assign="=>|:|=" - local comments="#" - local string="\"'" - if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then - endline='#' - fi - if [[ "$ext" =~ ^ini|env$ ]]; then - comments="[;#]" - fi - if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then - comments="//" - fi - local list='\[\s*['$string']?\w+['$string']?\]' - local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' - var_part+="[$string]?${key}[$string]?" - var_part+='\s*\]?\s*' - var_part+="($assign)" - var_part+='\s*' - - # Extract the part after assignation sign - local expression_with_comment="$((tail +$after_line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" - if [[ "$expression_with_comment" == "YNH_NULL" ]]; then - set -o xtrace # set -x - return 1 - fi - local value_line_number="$(tail +$after_line_number ${file} | grep -m1 -n -i -P $var_part'\K.*$' | cut -d: -f1)" - value_line_number=$((after_line_number + value_line_number)) - local range="${after_line_number},${value_line_number} " - - # Remove comments if needed - local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" - endline=${expression_with_comment#"$expression"} - endline="$(echo "$endline" | sed 's/\\/\\\\/g')" - value="$(echo "$value" | sed 's/\\/\\\\/g')" - value=${value//&/"\&"} - local first_char="${expression:0:1}" - delimiter=$'\001' - if [[ "$first_char" == '"' ]]; then - # \ and sed is quite complex you need 2 \\ to get one in a sed - # So we need \\\\ to go through 2 sed - value="$(echo "$value" | sed 's/"/\\\\"/g')" - sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file} - elif [[ "$first_char" == "'" ]]; then - # \ and sed is quite complex you need 2 \\ to get one in a sed - # However double quotes implies to double \\ to - # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str - value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" - sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file} - else - if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]]; then - value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' - fi - if [[ "$ext" =~ ^yaml|yml$ ]]; then - value=" $value" - fi - sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file} - fi - set -o xtrace # set -x -} - -# Render templates with Jinja2 -# -# [internal] -# -# Attention : Variables should be exported before calling this helper to be -# accessible inside templates. -# -# usage: ynh_render_template some_template output_path -# | arg: some_template - Template file to be rendered -# | arg: output_path - The path where the output will be redirected to -ynh_render_template() { - local template_path=$1 - local output_path=$2 - mkdir -p "$(dirname $output_path)" - # Taken from https://stackoverflow.com/a/35009576 - python3 -c 'import os, sys, jinja2; sys.stdout.write( - jinja2.Template(sys.stdin.read() - ).render(os.environ));' <$template_path >$output_path -} - _acceptable_path_to_delete() { local file=$1 From 80f07a9974e89445cdcaada1bb962d7e7f68173b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 May 2024 00:09:00 +0200 Subject: [PATCH 024/146] helpers 2.1: ynh_if_upgrading_from_... -> ynh_app_upgrading_from_... --- helpers/helpers.v2.1.d/utils | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index dadc478c3..83acad863 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -66,8 +66,8 @@ ynh_abort_if_errors() { trap ynh_exit_properly EXIT # Capturing exit signals on shell script } -# When running an app script with packaging format >= 2, auto-enable ynh_abort_if_errors except for remove script -if [[ "${YNH_CONTEXT:-}" != "regenconf" ]] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 && [[ ${YNH_APP_ACTION} != "remove" ]] +# When running an app script, auto-enable ynh_abort_if_errors except for remove script +if [[ "${YNH_CONTEXT:-}" != "regenconf" ]] && [[ "${YNH_APP_ACTION}" != "remove" ]] then ynh_abort_if_errors fi @@ -505,10 +505,10 @@ ynh_check_app_version_changed() { # Compare the current package version is strictly lower than another version given as an argument # -# example: if ynh_if_upgrading_from_version_prior_to 2.3.2~ynh1; then ... +# example: if ynh_app_upgrading_from_version_prior_to 2.3.2~ynh1; then ... # # Requires YunoHost version 11.2 or higher. -ynh_if_upgrading_from_version_prior_to() { +ynh_app_upgrading_from_version_prior_to() { local version=$1 [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" @@ -517,10 +517,10 @@ ynh_if_upgrading_from_version_prior_to() { # Compare the current package version is lower or equal to another version given as an argument # -# example: if ynh_if_upgrading_from_version_prior_or_equal_to 2.3.2~ynh1; then ... +# example: if ynh_app_upgrading_from_version_prior_or_equal_to 2.3.2~ynh1; then ... # # Requires YunoHost version 11.2 or higher. -ynh_if_upgrading_from_version_prior_or_equal_to() { +ynh_app_upgrading_from_version_prior_or_equal_to() { local version=$1 [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" From 3a1c8287b490b65e74e8bb21889c8129c2b2afee Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Jun 2024 15:08:26 +0200 Subject: [PATCH 025/146] helpers 2.1: merge new --jinja option for templating --- helpers/helpers.v2.1.d/templating | 62 +++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index 9ba6f7cff..b70b73e97 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -4,12 +4,17 @@ # # usage: ynh_add_config --template="template" --destination="destination" # | arg: -t, --template= - Template config file to use -# | arg: -d, --destination= - Destination of the config file +# | arg: -d, --destination= - Destination of the config file +# | arg: -j, --jinja - Use jinja template instead of legacy __MY_VAR__ # # examples: # ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" +# ynh_add_config --jinja --template="config.j2" --destination="$install_dir/config" use the template file "../conf/config.j2" # ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" # +## +## How it works in "legacy" mode +## # The template can be by default the name of a file in the conf directory # of a YunoHost Package, a relative path or an absolute path. # @@ -19,8 +24,6 @@ # ``` # __PATH__ by $path_url # __USER__ by $app -# __FINALPATH__ by $final_path -# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) # __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH # ``` # And any dynamic variables that should be defined before calling this helper like: @@ -31,6 +34,37 @@ # __VAR_2__ by $var_2 # ``` # +## +## When --jinja is enabled +## +# For a full documentation of the template you can refer to: https://jinja.palletsprojects.com/en/3.1.x/templates/ +# In Yunohost context there are no really some specificity except that all variable passed are of type string. +# So here are some example of recommended usage: +# +# If you need a conditional block +# +# {% if should_my_block_be_shown == 'true' %} +# ... +# {% endif %} +# +# or +# +# {% if should_my_block_be_shown == '1' %} +# ... +# {% endif %} +# +# If you need to iterate with loop: +# +# {% for yolo in var_with_multiline_value.splitlines() %} +# ... +# {% endfor %} +# +# or +# +# {% for jail in my_var_with_coma.split(',') %} +# ... +# {% endfor %} +# # The helper will verify the checksum and backup the destination file # if it's different before applying the new template. # @@ -40,10 +74,12 @@ # Requires YunoHost version 4.1.0 or higher. ynh_add_config() { # ============ Argument parsing ============= - local -A args_array=([t]=template= [d]=destination=) + local -A args_array=([t]=template= [d]=destination= [j]=jinja) local template local destination + local jinja ynh_handle_getopts_args "$@" + jinja="${jinja:-0}" # =========================================== local template_path @@ -62,14 +98,20 @@ ynh_add_config() { # created a file beforehand to have control over it # (cp won't overwrite ownership / modes by default...) touch $destination - chown root:root $destination chmod 640 $destination - - cp -f "$template_path" "$destination" - _ynh_apply_default_permissions $destination - ynh_replace_vars --file="$destination" + if [[ "$jinja" == 1 ]] + then + # This is ran in a subshell such that the "export" does not "contaminate" the main process + ( + export $(compgen -v) + j2 "$template_path" -f env -o $destination + ) + else + cp -f "$template_path" "$destination" + ynh_replace_vars --file="$destination" + fi ynh_store_file_checksum --file="$destination" } @@ -85,8 +127,6 @@ ynh_add_config() { # that should be defined before calling this helper : # __PATH__ by $path_url # __USER__ by $app -# __FINALPATH__ by $final_path -# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) # __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH # # And any dynamic variables that should be defined before calling this helper like: From 727b0e093a9b305d9aa37b152563418704c6dab7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 8 Jun 2024 16:56:23 +0200 Subject: [PATCH 026/146] helpers 2.1: rename ynh_secure_remove --file to ynh_safe_rm --target --- helpers/helpers.v2.1.d/apt | 6 +++--- helpers/helpers.v2.1.d/backup | 2 +- helpers/helpers.v2.1.d/config | 2 +- helpers/helpers.v2.1.d/fail2ban | 4 ++-- helpers/helpers.v2.1.d/go | 4 ++-- helpers/helpers.v2.1.d/logging | 2 +- helpers/helpers.v2.1.d/mongodb | 4 ++-- helpers/helpers.v2.1.d/nginx | 4 ++-- helpers/helpers.v2.1.d/nodejs | 4 ++-- helpers/helpers.v2.1.d/php | 4 ++-- helpers/helpers.v2.1.d/ruby | 6 +++--- helpers/helpers.v2.1.d/systemd | 4 ++-- helpers/helpers.v2.1.d/utils | 34 ++++++++++++++++----------------- 13 files changed, 40 insertions(+), 40 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 1309042f8..2e13f1186 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -481,10 +481,10 @@ EOF # Requires YunoHost version 3.8.1 or higher. ynh_remove_extra_repo() { - ynh_secure_remove --file="/etc/apt/sources.list.d/$app.list" - ynh_secure_remove --file="/etc/apt/preferences.d/$app" + ynh_safe_rm --target="/etc/apt/sources.list.d/$app.list" + ynh_safe_rm --target="/etc/apt/preferences.d/$app" if [ -e /etc/apt/trusted.gpg.d/$app.gpg ]; then - ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$app.gpg" + ynh_safe_rm --target="/etc/apt/trusted.gpg.d/$app.gpg" fi ynh_package_update } diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index bcf322914..6b147ed6a 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -265,7 +265,7 @@ ynh_restore_file() { mkdir --parents "$(dirname "$backup_file")" mv "${dest_path}" "$backup_file" # Move the current file or directory else - ynh_secure_remove --file=${dest_path} + ynh_safe_rm --target=${dest_path} fi fi diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index a1784aff2..4d4b74465 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -76,7 +76,7 @@ _ynh_app_config_apply_one() { local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]]; then ynh_backup_if_checksum_is_different --file="$bind_file" - ynh_secure_remove --file="$bind_file" + ynh_safe_rm --target="$bind_file" ynh_delete_file_checksum --file="$bind_file" ynh_print_info --message="File '$bind_file' removed" else diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index 12c579db6..8e7e34b7a 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -129,7 +129,7 @@ ignoreregex = # # Requires YunoHost version 3.5.0 or higher. ynh_remove_fail2ban_config() { - ynh_secure_remove --file="/etc/fail2ban/jail.d/$app.conf" - ynh_secure_remove --file="/etc/fail2ban/filter.d/$app.conf" + ynh_safe_rm --target="/etc/fail2ban/jail.d/$app.conf" + ynh_safe_rm --target="/etc/fail2ban/filter.d/$app.conf" ynh_systemd_action --service_name=fail2ban --action=reload } diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index cf8290390..239bf848f 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -240,7 +240,7 @@ ynh_cleanup_go () { then # Remove goenv environment configuration ynh_print_info --message="Removing of goenv" - ynh_secure_remove --file="$goenv_install_dir" - ynh_secure_remove --file="/etc/profile.d/goenv.sh" + ynh_safe_rm --target="$goenv_install_dir" + ynh_safe_rm --target="/etc/profile.d/goenv.sh" fi } diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index 1e4b15871..4af30af85 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -197,7 +197,7 @@ ynh_exec_and_print_stderr_only_if_error() { "$@" 2> "$logfile" || rc="$?" if (( rc != 0 )); then ynh_exec_warn cat "$logfile" - ynh_secure_remove --file="$logfile" + ynh_safe_rm --target="$logfile" return "$rc" fi } diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index c86369590..608e7f055 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -339,7 +339,7 @@ ynh_remove_mongo() { mongodb_servicename=mongod # Remove the mongodb service yunohost service remove $mongodb_servicename - ynh_secure_remove --file="/var/lib/mongodb" - ynh_secure_remove --file="/var/log/mongodb" + ynh_safe_rm --target="/var/lib/mongodb" + ynh_safe_rm --target="/var/log/mongodb" fi } diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index 600c70a49..b9f48dafe 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -39,7 +39,7 @@ ynh_add_nginx_config() { # # Requires YunoHost version 2.7.2 or higher. ynh_remove_nginx_config() { - ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf" + ynh_safe_rm --target="/etc/nginx/conf.d/$domain.d/$app.conf" ynh_systemd_action --service_name=nginx --action=reload } @@ -58,7 +58,7 @@ ynh_change_url_nginx_config() { local old_nginx_conf_path=/etc/nginx/conf.d/$old_domain.d/$app.conf ynh_backup_if_checksum_is_different --file="$old_nginx_conf_path" ynh_delete_file_checksum --file="$old_nginx_conf_path" - ynh_secure_remove --file="$old_nginx_conf_path" + ynh_safe_rm --target="$old_nginx_conf_path" # Regen the nginx conf ynh_add_nginx_config diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index 74491a590..72afd6b7a 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -176,8 +176,8 @@ ynh_remove_nodejs() { # If no other app uses n, remove n if [ ! -s "$n_install_dir/ynh_app_version" ]; then - ynh_secure_remove --file="$n_install_dir" - ynh_secure_remove --file="/usr/local/n" + ynh_safe_rm --target="$n_install_dir" + ynh_safe_rm --target="/usr/local/n" sed --in-place "/N_PREFIX/d" /root/.bashrc rm --force /etc/cron.daily/node_update fi diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 7371f585e..a48b93ea5 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -127,7 +127,7 @@ pm.process_idle_timeout = 10s # Validate that the new php conf doesn't break php-fpm entirely if ! php-fpm${phpversion} --test 2>/dev/null; then php-fpm${phpversion} --test || true - ynh_secure_remove --file="$finalphpconf" + ynh_safe_rm --target="$finalphpconf" ynh_die --message="The new configuration broke php-fpm?" fi ynh_systemd_action --service_name=$fpm_service --action=reload @@ -142,7 +142,7 @@ ynh_remove_fpm_config() { local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local fpm_service=$(ynh_app_setting_get --key=fpm_service) - ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" + ynh_safe_rm --target="$fpm_config_dir/pool.d/$app.conf" ynh_systemd_action --service_name=$fpm_service --action=reload } diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 99fa6f016..1b4f193bb 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -123,7 +123,7 @@ ynh_install_ruby () { else ynh_print_info --message="Reinstalling rbenv..." cd .. - ynh_secure_remove --file=$rbenv_install_dir + ynh_safe_rm --target=$rbenv_install_dir mkdir -p $rbenv_install_dir cd $rbenv_install_dir git init -q @@ -291,8 +291,8 @@ ynh_cleanup_ruby () { then # Remove rbenv environment configuration ynh_print_info --message="Removing rbenv" - ynh_secure_remove --file="$rbenv_install_dir" - ynh_secure_remove --file="/etc/profile.d/rbenv.sh" + ynh_safe_rm --target="$rbenv_install_dir" + ynh_safe_rm --target="/etc/profile.d/rbenv.sh" fi } diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index d1742c23e..748c31cfa 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -46,7 +46,7 @@ ynh_remove_systemd_config() { if [ -e "$finalsystemdconf" ]; then ynh_systemd_action --service_name=$service --action=stop systemctl disable $service --quiet - ynh_secure_remove --file="$finalsystemdconf" + ynh_safe_rm --target="$finalsystemdconf" systemctl daemon-reload fi } @@ -181,6 +181,6 @@ ynh_clean_check_starting() { kill -SIGTERM $pid_tail 2>&1 fi if [ -n "${templog:-}" ]; then - ynh_secure_remove --file="$templog" 2>&1 + ynh_safe_rm --target="$templog" 2>&1 fi } diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 83acad863..5974c7d31 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -259,7 +259,7 @@ ynh_setup_source() { fi if [ "$full_replace" -eq 1 ]; then - ynh_secure_remove --file="$dest_dir" + ynh_safe_rm --target="$dest_dir" fi # Extract source into the app dir @@ -288,11 +288,11 @@ ynh_setup_source() { local tmp_dir=$(mktemp --directory) unzip -quo $src_filename -d "$tmp_dir" cp --archive $tmp_dir/*/. "$dest_dir" - ynh_secure_remove --file="$tmp_dir" + ynh_safe_rm --target="$tmp_dir" else unzip -quo $src_filename -d "$dest_dir" fi - ynh_secure_remove --file="$src_filename" + ynh_safe_rm --target="$src_filename" else local strip="" if [ "$src_in_subdir" != "false" ]; then @@ -308,7 +308,7 @@ ynh_setup_source() { else ynh_die --message="Archive format unrecognized." fi - ynh_secure_remove --file="$src_filename" + ynh_safe_rm --target="$src_filename" fi # Apply patches @@ -432,30 +432,30 @@ _acceptable_path_to_delete() { # Remove a file or a directory securely # -# usage: ynh_secure_remove --file=path_to_remove -# | arg: -f, --file= - File or directory to remove +# usage: ynh_safe_rm --target=path_to_remove +# | arg: -t, --target= - File or directory to remove # # Requires YunoHost version 2.6.4 or higher. -ynh_secure_remove() { +ynh_safe_rm() { # ============ Argument parsing ============= - local -A args_array=([f]=file=) - local file + local -A args_array=([t]=target=) + local target ynh_handle_getopts_args "$@" # =========================================== set +o xtrace # set +x if [ $# -ge 2 ]; then - ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." + ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_safe_rm but it will be ignored... Use this helper with one argument at time." fi - if [[ -z "$file" ]]; then - ynh_print_warn --message="ynh_secure_remove called with empty argument, ignoring." - elif [[ ! -e $file ]]; then - ynh_print_info --message="'$file' wasn't deleted because it doesn't exist." - elif ! _acceptable_path_to_delete "$file"; then - ynh_print_warn --message="Not deleting '$file' because it is not an acceptable path to delete." + if [[ -z "$target" ]]; then + ynh_print_warn --message="ynh_safe_rm called with empty argument, ignoring." + elif [[ ! -e $target ]]; then + ynh_print_info --message="'$target' wasn't deleted because it doesn't exist." + elif ! _acceptable_path_to_delete "$target"; then + ynh_print_warn --message="Not deleting '$target' because it is not an acceptable path to delete." else - rm --recursive "$file" + rm --recursive "$target" fi set -o xtrace # set -x From 5c461d6058fe6205e8657ebaa9adae07eb94d848 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 8 Jun 2024 17:30:41 +0200 Subject: [PATCH 027/146] helpers2.1: import go changes from v1 to v2.1 --- helpers/helpers.v2.1.d/go | 81 ++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 239bf848f..d6b74aaf9 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -110,43 +110,36 @@ ynh_install_go () { test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv # Install or update goenv - goenv="$(command -v goenv $goenv_install_dir/bin/goenv | head -1)" - if [ -n "$goenv" ]; then - ynh_print_info --message="goenv already seems installed in \`$goenv'." - pushd "${goenv%/*/*}" - if git remote -v 2>/dev/null | grep "https://github.com/syndbg/goenv.git"; then - echo "Trying to update with Git..." - git pull -q --tags origin master - cd .. - ynh_go_try_bash_extension - fi - popd - else - ynh_print_info --message="Installing goenv with Git..." - mkdir -p $goenv_install_dir - pushd $goenv_install_dir + mkdir -p $goenv_install_dir + pushd "$goenv_install_dir" + if ! [ -x "$goenv_install_dir/bin/goenv" ]; then + ynh_print_info --message="Downloading goenv..." git init -q - git remote add -f -t master origin https://github.com/syndbg/goenv.git > /dev/null 2>&1 - git checkout -q -b master origin/master - ynh_go_try_bash_extension - goenv=$goenv_install_dir/bin/goenv - popd - fi + git remote add origin https://github.com/syndbg/goenv.git + else + ynh_print_info --message="Updating goenv..." + fi + git fetch -q --tags --prune origin + local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout -q "$git_latest_tag" + ynh_go_try_bash_extension + goenv=$goenv_install_dir/bin/goenv + popd - goenv_latest="$(command -v "$goenv_install_dir"/plugins/*/bin/goenv-latest goenv-latest | head -1)" - if [ -n "$goenv_latest" ]; then - ynh_print_info --message="\`goenv latest' command already available in \`$goenv_latest'." - pushd "${goenv_latest%/*/*}" - if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then - ynh_print_info --message="Trying to update xxenv-latest with git..." - git pull -q origin master - fi - popd - else - ynh_print_info --message="Installing xxenv-latest with Git..." - mkdir -p "${goenv_install_dir}/plugins" - git clone -q https://github.com/momo-lab/xxenv-latest.git "${goenv_install_dir}/plugins/xxenv-latest" - fi + # Install or update xxenv-latest + mkdir -p "$goenv_install_dir/plugins/xxenv-latest" + pushd "$goenv_install_dir/plugins/xxenv-latest" + if ! [ -x "$goenv_install_dir/plugins/xxenv-latest/bin/goenv-latest" ]; then + ynh_print_info --message="Downloading xxenv-latest..." + git init -q + git remote add origin https://github.com/momo-lab/xxenv-latest.git + else + ynh_print_info --message="Updating xxenv-latest..." + fi + git fetch -q --tags --prune origin + local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout -q "$git_latest_tag" + popd # Enable caching mkdir -p "${goenv_install_dir}/cache" @@ -161,12 +154,12 @@ ynh_install_go () { test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go # Install the requested version of Go - local final_go_version=$(goenv latest --print $go_version) + local final_go_version=$(goenv latest --print "$go_version") ynh_print_info --message="Installation of Go-$final_go_version" - goenv install --skip-existing $final_go_version + goenv install --skip-existing "$final_go_version" # Store go_version into the config of this app - ynh_app_setting_set --key=go_version --value=$final_go_version + ynh_app_setting_set --app="$app" --key="go_version" --value="$final_go_version" # Cleanup Go versions ynh_cleanup_go @@ -188,7 +181,7 @@ eval \"\$(goenv init -)\" # # usage: ynh_remove_go ynh_remove_go () { - local go_version=$(ynh_app_setting_get --key=go_version) + local go_version=$(ynh_app_setting_get --key="go_version") # Load goenv path in PATH local CLEAR_PATH="$goenv_install_dir/bin:$PATH" @@ -197,7 +190,7 @@ ynh_remove_go () { PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') # Remove the line for this app - ynh_app_setting_delete --key=go_version + ynh_app_setting_delete --key="go_version" # Cleanup Go versions ynh_cleanup_go @@ -223,7 +216,7 @@ ynh_cleanup_go () { required_go_versions="${installed_app_go_version}\n${required_go_versions}" fi done - + # Remove no more needed Go versions local installed_go_versions=$(goenv versions --bare --skip-aliases | grep -Ev '/') for installed_go_version in $installed_go_versions @@ -231,7 +224,7 @@ ynh_cleanup_go () { if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1` then ynh_print_info --message="Removing of Go-$installed_go_version" - $goenv_install_dir/bin/goenv uninstall --force $installed_go_version + $goenv_install_dir/bin/goenv uninstall --force "$installed_go_version" fi done @@ -240,7 +233,7 @@ ynh_cleanup_go () { then # Remove goenv environment configuration ynh_print_info --message="Removing of goenv" - ynh_safe_rm --target="$goenv_install_dir" - ynh_safe_rm --target="/etc/profile.d/goenv.sh" + ynh_secure_remove --file="$goenv_install_dir" + ynh_secure_remove --file="/etc/profile.d/goenv.sh" fi } From 8117f438d4c9dd2ba67f6c84de9133e90cecb495 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 8 Jun 2024 17:36:35 +0200 Subject: [PATCH 028/146] helpers2.1: vendor is a symlink to the folder in parent --- .../vendor/docker-image-extract/LICENSE | 19 - .../vendor/docker-image-extract/README.md | 1 - .../docker-image-extract/docker-image-extract | 288 --- helpers/helpers.v2.1.d/vendor/n/LICENSE | 21 - helpers/helpers.v2.1.d/vendor/n/README.md | 1 - helpers/helpers.v2.1.d/vendor/n/n | 1713 ----------------- 6 files changed, 2043 deletions(-) delete mode 100644 helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE delete mode 100644 helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md delete mode 100755 helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract delete mode 100644 helpers/helpers.v2.1.d/vendor/n/LICENSE delete mode 100644 helpers/helpers.v2.1.d/vendor/n/README.md delete mode 100755 helpers/helpers.v2.1.d/vendor/n/n diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE b/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE deleted file mode 100644 index 986360f1a..000000000 --- a/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2020-2023, Jeremy Lin - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md b/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md deleted file mode 100644 index 4c4fa301f..000000000 --- a/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md +++ /dev/null @@ -1 +0,0 @@ -This is taken from https://github.com/jjlin/docker-image-extract, under MIT license. \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract b/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract deleted file mode 100755 index b5dfdb7a7..000000000 --- a/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract +++ /dev/null @@ -1,288 +0,0 @@ -#!/bin/sh -# -# This script pulls and extracts all files from an image in Docker Hub. -# -# Copyright (c) 2020-2023, Jeremy Lin -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -PLATFORM_DEFAULT="linux/amd64" -PLATFORM="${PLATFORM_DEFAULT}" -OUT_DIR="./output" - -usage() { - echo "This script pulls and extracts all files from an image in Docker Hub." - echo - echo "$0 [OPTIONS...] IMAGE[:REF]" - echo - echo "IMAGE can be a community user image (like 'some-user/some-image') or a" - echo "Docker official image (like 'hello-world', which contains no '/')." - echo - echo "REF is either a tag name or a full SHA-256 image digest (with a 'sha256:' prefix)." - echo "The default ref is the 'latest' tag." - echo - echo "Options:" - echo - echo " -p PLATFORM Pull image for the specified platform (default: ${PLATFORM})" - echo " For a given image on Docker Hub, the 'Tags' tab lists the" - echo " platforms supported for that image." - echo " -o OUT_DIR Extract image to the specified output dir (default: ${OUT_DIR})" - echo " -h Show help with usage examples" -} - -usage_detailed() { - usage - echo - echo "Examples:" - echo - echo "# Pull and extract all files in the 'hello-world' image tagged 'latest'." - echo "\$ $0 hello-world:latest" - echo - echo "# Same as above; ref defaults to the 'latest' tag." - echo "\$ $0 hello-world" - echo - echo "# Pull the 'hello-world' image for the 'linux/arm64/v8' platform." - echo "\$ $0 -p linux/arm64/v8 hello-world" - echo - echo "# Pull an image by digest." - echo "\$ $0 hello-world:sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042" -} - -if [ $# -eq 0 ]; then - usage_detailed - exit 0 -fi - -while getopts ':ho:p:' opt; do - case $opt in - o) - OUT_DIR="${OPTARG}" - ;; - p) - PLATFORM="${OPTARG}" - ;; - h) - usage_detailed - exit 0 - ;; - \?) - echo "ERROR: Invalid option '-$OPTARG'." - echo - usage - exit 1 - ;; - \:) echo "ERROR: Argument required for option '-$OPTARG'." - echo - usage - exit 1 - ;; - esac -done -shift $(($OPTIND - 1)) - -if [ $# -eq 0 ]; then - echo "ERROR: Image to pull must be specified." - echo - usage - exit 1 -fi - -if [ -e "${OUT_DIR}" ]; then - if [ -d "${OUT_DIR}" ]; then - echo "WARNING: Output dir already exists. If it contains a previous extracted image," - echo "there may be errors when trying to overwrite files with read-only permissions." - echo - else - echo "ERROR: Output dir already exists, but is not a directory." - exit 1 - fi -fi - -have_curl() { - command -v curl >/dev/null -} - -have_wget() { - command -v wget >/dev/null -} - -if ! have_curl && ! have_wget; then - echo "This script requires either curl or wget." - exit 1 -fi - -image_spec="$1" -image="${image_spec%%:*}" -if [ "${image#*/}" = "${image}" ]; then - # Docker official images are in the 'library' namespace. - image="library/${image}" -fi -ref="${image_spec#*:}" -if [ "${ref}" = "${image_spec}" ]; then - echo "Defaulting ref to tag 'latest'..." - ref=latest -fi - -# Split platform (OS/arch/variant) into separate variables. -# A platform specifier doesn't always include the `variant` component. -OLD_IFS="${IFS}" -IFS=/ read -r OS ARCH VARIANT <":"" (assumes key/val won't contain double quotes). - # The colon may have whitespace on either side. - grep -o "\"${key}\"[[:space:]]*:[[:space:]]*\"[^\"]\+\"" | - # Extract just by deleting the last '"', and then greedily deleting - # everything up to '"'. - sed -e 's/"$//' -e 's/.*"//' -} - -# Fetch a URL to stdout. Up to two header arguments may be specified: -# -# fetch [name1: value1] [name2: value2] -# -fetch() { - if have_curl; then - if [ $# -eq 2 ]; then - set -- -H "$2" "$1" - elif [ $# -eq 3 ]; then - set -- -H "$2" -H "$3" "$1" - fi - curl -sSL "$@" - else - if [ $# -eq 2 ]; then - set -- --header "$2" "$1" - elif [ $# -eq 3 ]; then - set -- --header "$2" --header "$3" "$1" - fi - wget -qO- "$@" - fi -} - -# https://docs.docker.com/docker-hub/api/latest/#tag/repositories -manifest_list_url="https://hub.docker.com/v2/repositories/${image}/tags/${ref}" - -# If the ref is already a SHA-256 image digest, then we don't need to look up anything. -if [ -z "${ref##sha256:*}" ]; then - digest="${ref}" -else - echo "Getting multi-arch manifest list..." - NL=' -' - digest=$(fetch "${manifest_list_url}" | - # Break up the single-line JSON output into separate lines by adding - # newlines before and after the chars '[', ']', '{', and '}'. - # This uses the \${NL} syntax because some BSD variants of sed don't - # support \n syntax in the replacement string, but instead require - # a literal newline preceded by a backslash. - sed -e 's/\([][{}]\)/\'"${NL}"'\1\'"${NL}"'/g' | - # Extract the "images":[...] list. - sed -n '/"images":/,/]/ p' | - # Each image's details are now on a separate line, e.g. - # "architecture":"arm64","features":"","variant":"v8","digest":"sha256:054c85801c4cb41511b176eb0bf13a2c4bbd41611ddd70594ec3315e88813524","os":"linux","os_features":"","os_version":null,"size":828724,"status":"active","last_pulled":"2022-09-02T22:46:48.240632Z","last_pushed":"2022-09-02T00:42:45.69226Z" - # The image details are interspersed with lines of stray punctuation, - # so grep for an arbitrary string that must be in these lines. - grep architecture | - # Search for an image that matches the platform. - while read -r image; do - # Arch is probably most likely to be unique, so check that first. - arch="$(echo ${image} | extract 'architecture')" - if [ "${arch}" != "${ARCH}" ]; then continue; fi - - os="$(echo ${image} | extract 'os')" - if [ "${os}" != "${OS}" ]; then continue; fi - - variant="$(echo ${image} | extract 'variant')" - if [ "${variant}" = "${VARIANT}" ]; then - echo ${image} | extract 'digest' - break - fi - done) - - if [ -n "${digest}" ]; then - echo "Platform ${PLATFORM} resolved to '${digest}'..." - else - echo "No image digest found. Verify that the image, ref, and platform are valid." - exit 1 - fi -fi - -# https://docs.docker.com/registry/spec/auth/token/#how-to-authenticate -api_token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" - -# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-an-image-manifest -manifest_url="https://registry-1.docker.io/v2/${image}/manifests/${digest}" - -# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-a-layer -blobs_base_url="https://registry-1.docker.io/v2/${image}/blobs" - -echo "Getting API token..." -token=$(fetch "${api_token_url}" | extract 'token') -auth_header="Authorization: Bearer $token" - -# https://github.com/distribution/distribution/blob/main/docs/spec/manifest-v2-2.md -docker_manifest_v2="application/vnd.docker.distribution.manifest.v2+json" - -# https://github.com/opencontainers/image-spec/blob/main/manifest.md -oci_manifest_v1="application/vnd.oci.image.manifest.v1+json" - -# Docker Hub can return either type of manifest format. Most images seem to -# use the Docker format for now, but the OCI format will likely become more -# common as features that require that format become enabled by default -# (e.g., https://github.com/docker/build-push-action/releases/tag/v3.3.0). -accept_header="Accept: ${docker_manifest_v2},${oci_manifest_v1}" - -echo "Getting image manifest for $image:$ref..." -layers=$(fetch "${manifest_url}" "${auth_header}" "${accept_header}" | - # Extract `digest` values only after the `layers` section appears. - sed -n '/"layers":/,$ p' | - extract 'digest') - -if [ -z "${layers}" ]; then - echo "No layers returned. Verify that the image and ref are valid." - exit 1 -fi - -mkdir -p "${OUT_DIR}" - -for layer in $layers; do - hash="${layer#sha256:}" - echo "Fetching and extracting layer ${hash}..." - fetch "${blobs_base_url}/${layer}" "${auth_header}" | gzip -d | tar -C "${OUT_DIR}" -xf - - # Ref: https://github.com/moby/moby/blob/master/image/spec/v1.2.md#creating-an-image-filesystem-changeset - # https://github.com/moby/moby/blob/master/pkg/archive/whiteouts.go - # Search for "whiteout" files to indicate files deleted in this layer. - OLD_IFS="${IFS}" - find "${OUT_DIR}" -name '.wh.*' | while IFS= read -r f; do - dir="${f%/*}" - wh_file="${f##*/}" - file="${wh_file#.wh.}" - # Delete both the whiteout file and the whited-out file. - rm -rf "${dir}/${wh_file}" "${dir}/${file}" - done - IFS="${OLD_IFS}" -done - -echo "Image contents extracted into ${OUT_DIR}." \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/n/LICENSE b/helpers/helpers.v2.1.d/vendor/n/LICENSE deleted file mode 100644 index 8e04e8467..000000000 --- a/helpers/helpers.v2.1.d/vendor/n/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2018 TJ Holowaychuk - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/helpers/helpers.v2.1.d/vendor/n/README.md b/helpers/helpers.v2.1.d/vendor/n/README.md deleted file mode 100644 index 9a29a3936..000000000 --- a/helpers/helpers.v2.1.d/vendor/n/README.md +++ /dev/null @@ -1 +0,0 @@ -This is taken from https://github.com/tj/n/ diff --git a/helpers/helpers.v2.1.d/vendor/n/n b/helpers/helpers.v2.1.d/vendor/n/n deleted file mode 100755 index 86b6a0fa9..000000000 --- a/helpers/helpers.v2.1.d/vendor/n/n +++ /dev/null @@ -1,1713 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2155 -# Disabled "Declare and assign separately to avoid masking return values": https://github.com/koalaman/shellcheck/wiki/SC2155 - -# -# log -# - -log() { - printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" -} - -# -# verbose_log -# Can suppress with --quiet. -# Like log but to stderr rather than stdout, so can also be used from "display" routines. -# - -verbose_log() { - if [[ "${SHOW_VERBOSE_LOG}" == "true" ]]; then - >&2 printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" - fi -} - -# -# Exit with the given -# - -abort() { - >&2 printf "\n ${SGR_RED}Error: %s${SGR_RESET}\n\n" "$*" && exit 1 -} - -# -# Synopsis: trace message ... -# Debugging output to stderr, not used in production code. -# - -function trace() { - >&2 printf "trace: %s\n" "$*" -} - -# -# Synopsis: echo_red message ... -# Highlight message in colour (on stdout). -# - -function echo_red() { - printf "${SGR_RED}%s${SGR_RESET}\n" "$*" -} - -# -# Synopsis: n_grep -# grep wrapper to ensure consistent grep options and circumvent aliases. -# - -function n_grep() { - GREP_OPTIONS='' command grep "$@" -} - -# -# Setup and state -# - -VERSION="v9.2.3" - -N_PREFIX="${N_PREFIX-/usr/local}" -N_PREFIX=${N_PREFIX%/} -readonly N_PREFIX - -N_CACHE_PREFIX="${N_CACHE_PREFIX-${N_PREFIX}}" -N_CACHE_PREFIX=${N_CACHE_PREFIX%/} -CACHE_DIR="${N_CACHE_PREFIX}/n/versions" -readonly N_CACHE_PREFIX CACHE_DIR - -N_NODE_MIRROR=${N_NODE_MIRROR:-${NODE_MIRROR:-https://nodejs.org/dist}} -N_NODE_MIRROR=${N_NODE_MIRROR%/} -readonly N_NODE_MIRROR - -N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR:-https://nodejs.org/download} -N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR%/} -readonly N_NODE_DOWNLOAD_MIRROR - -# Using xz instead of gzip is enabled by default, if xz compatibility checks pass. -# User may set N_USE_XZ to 0 to disable, or set to anything else to enable. -# May also be overridden by command line flags. - -# Normalise external values to true/false -if [[ "${N_USE_XZ}" = "0" ]]; then - N_USE_XZ="false" -elif [[ -n "${N_USE_XZ+defined}" ]]; then - N_USE_XZ="true" -fi -# Not setting to readonly. Overriden by CLI flags, and update_xz_settings_for_version. - -N_MAX_REMOTE_MATCHES=${N_MAX_REMOTE_MATCHES:-20} -# modified by update_mirror_settings_for_version -g_mirror_url=${N_NODE_MIRROR} -g_mirror_folder_name="node" - -# Options for curl and wget. -# Defining commands in variables is fraught (https://mywiki.wooledge.org/BashFAQ/050) -# but we can follow the simple case and store arguments in an array. - -GET_SHOWS_PROGRESS="false" -# --location to follow redirects -# --fail to avoid happily downloading error page from web server for 404 et al -# --show-error to show why failed (on stderr) -CURL_OPTIONS=( "--location" "--fail" "--show-error" ) -if [[ -t 1 ]]; then - CURL_OPTIONS+=( "--progress-bar" ) - command -v curl &> /dev/null && GET_SHOWS_PROGRESS="true" -else - CURL_OPTIONS+=( "--silent" ) -fi -WGET_OPTIONS=( "-q" "-O-" ) - -# Legacy support using unprefixed env. No longer documented in README. -if [ -n "$HTTP_USER" ];then - if [ -z "$HTTP_PASSWORD" ]; then - abort "Must specify HTTP_PASSWORD when supplying HTTP_USER" - fi - CURL_OPTIONS+=( "-u $HTTP_USER:$HTTP_PASSWORD" ) - WGET_OPTIONS+=( "--http-password=$HTTP_PASSWORD" - "--http-user=$HTTP_USER" ) -elif [ -n "$HTTP_PASSWORD" ]; then - abort "Must specify HTTP_USER when supplying HTTP_PASSWORD" -fi - -# Set by set_active_node -g_active_node= - -# set by various lookups to allow mixed logging and return value from function, especially for engine and node -g_target_node= - -DOWNLOAD=false # set to opt-out of activate (install), and opt-in to download (run, exec) -ARCH= -SHOW_VERBOSE_LOG="true" -OFFLINE=false - -# ANSI escape codes -# https://en.wikipedia.org/wiki/ANSI_escape_code -# https://no-color.org -# https://bixense.com/clicolors - -USE_COLOR="true" -if [[ -n "${CLICOLOR_FORCE+defined}" && "${CLICOLOR_FORCE}" != "0" ]]; then - USE_COLOR="true" -elif [[ -n "${NO_COLOR+defined}" || "${CLICOLOR}" = "0" || ! -t 1 ]]; then - USE_COLOR="false" -fi -readonly USE_COLOR -# Select Graphic Rendition codes -if [[ "${USE_COLOR}" = "true" ]]; then - # KISS and use codes rather than tput, avoid dealing with missing tput or TERM. - readonly SGR_RESET="\033[0m" - readonly SGR_FAINT="\033[2m" - readonly SGR_RED="\033[31m" - readonly SGR_CYAN="\033[36m" -else - readonly SGR_RESET= - readonly SGR_FAINT= - readonly SGR_RED= - readonly SGR_CYAN= -fi - -# -# set_arch to override $(uname -a) -# - -set_arch() { - if test -n "$1"; then - ARCH="$1" - else - abort "missing -a|--arch value" - fi -} - -# -# Synopsis: set_insecure -# Globals modified: -# - CURL_OPTIONS -# - WGET_OPTIONS -# - -function set_insecure() { - CURL_OPTIONS+=( "--insecure" ) - WGET_OPTIONS+=( "--no-check-certificate" ) -} - -# -# Synposis: display_major_version numeric-version -# -display_major_version() { - local version=$1 - version="${version#v}" - version="${version%%.*}" - echo "${version}" -} - -# -# Synopsis: update_mirror_settings_for_version version -# e.g. means using download mirror and folder is nightly -# Globals modified: -# - g_mirror_url -# - g_mirror_folder_name -# - -function update_mirror_settings_for_version() { - if is_download_folder "$1" ; then - g_mirror_folder_name="$1" - g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" - elif is_download_version "$1"; then - [[ "$1" =~ ^([^/]+)/(.*) ]] - local remote_folder="${BASH_REMATCH[1]}" - g_mirror_folder_name="${remote_folder}" - g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" - fi -} - -# -# Synopsis: update_xz_settings_for_version numeric-version -# Globals modified: -# - N_USE_XZ -# - -function update_xz_settings_for_version() { - # tarballs in xz format were available in later version of iojs, but KISS and only use xz from v4. - if [[ "${N_USE_XZ}" = "true" ]]; then - local major_version="$(display_major_version "$1")" - if [[ "${major_version}" -lt 4 ]]; then - N_USE_XZ="false" - fi - fi -} - -# -# Synopsis: update_arch_settings_for_version numeric-version -# Globals modified: -# - ARCH -# - -function update_arch_settings_for_version() { - local tarball_platform="$(display_tarball_platform)" - if [[ -z "${ARCH}" && "${tarball_platform}" = "darwin-arm64" ]]; then - # First native builds were for v16, but can use x64 in rosetta for older versions. - local major_version="$(display_major_version "$1")" - if [[ "${major_version}" -lt 16 ]]; then - ARCH=x64 - fi - fi -} - -# -# Synopsis: is_lts_codename version -# - -function is_lts_codename() { - # https://github.com/nodejs/Release/blob/master/CODENAMES.md - # e.g. argon, Boron - [[ "$1" =~ ^([Aa]rgon|[Bb]oron|[Cc]arbon|[Dd]ubnium|[Ee]rbium|[Ff]ermium|[Gg]allium|[Hh]ydrogen|[Ii]ron|[Jj]od)$ ]] -} - -# -# Synopsis: is_download_folder version -# - -function is_download_folder() { - # e.g. nightly - [[ "$1" =~ ^(next-nightly|nightly|rc|release|test|v8-canary)$ ]] -} - -# -# Synopsis: is_download_version version -# - -function is_download_version() { - # e.g. nightly/, nightly/latest, nightly/v11 - if [[ "$1" =~ ^([^/]+)/(.*) ]]; then - local remote_folder="${BASH_REMATCH[1]}" - is_download_folder "${remote_folder}" - return - fi - return 2 -} - -# -# Synopsis: is_numeric_version version -# - -function is_numeric_version() { - # e.g. 6, v7.1, 8.11.3 - [[ "$1" =~ ^[v]{0,1}[0-9]+(\.[0-9]+){0,2}$ ]] -} - -# -# Synopsis: is_exact_numeric_version version -# - -function is_exact_numeric_version() { - # e.g. 6, v7.1, 8.11.3 - [[ "$1" =~ ^[v]{0,1}[0-9]+\.[0-9]+\.[0-9]+$ ]] -} - -# -# Synopsis: is_node_support_version version -# Reference: https://github.com/nodejs/package-maintenance/issues/236#issue-474783582 -# - -function is_node_support_version() { - [[ "$1" =~ ^(active|lts_active|lts_latest|lts|current|supported)$ ]] -} - -# -# Synopsis: display_latest_node_support_alias version -# Map aliases onto existing n aliases, current and lts -# - -function display_latest_node_support_alias() { - case "$1" in - "active") printf "current" ;; - "lts_active") printf "lts" ;; - "lts_latest") printf "lts" ;; - "lts") printf "lts" ;; - "current") printf "current" ;; - "supported") printf "current" ;; - *) printf "unexpected-version" - esac -} - -# -# Functions used when showing versions installed -# - -enter_fullscreen() { - # Set cursor to be invisible - tput civis 2> /dev/null - # Save screen contents - tput smcup 2> /dev/null - stty -echo -} - -leave_fullscreen() { - # Set cursor to normal - tput cnorm 2> /dev/null - # Restore screen contents - tput rmcup 2> /dev/null - stty echo -} - -handle_sigint() { - leave_fullscreen - S="$?" - kill 0 - exit $S -} - -handle_sigtstp() { - leave_fullscreen - kill -s SIGSTOP $$ -} - -# -# Output usage information. -# - -display_help() { - cat <<-EOF - -Usage: n [options] [COMMAND] [args] - -Commands: - - n Display downloaded Node.js versions and install selection - n latest Install the latest Node.js release (downloading if necessary) - n lts Install the latest LTS Node.js release (downloading if necessary) - n Install Node.js (downloading if necessary) - n install Install Node.js (downloading if necessary) - n run [args ...] Execute downloaded Node.js with [args ...] - n which Output path for downloaded node - n exec [args...] Execute command with modified PATH, so downloaded node and npm first - n rm Remove the given downloaded version(s) - n prune Remove all downloaded versions except the installed version - n --latest Output the latest Node.js version available - n --lts Output the latest LTS Node.js version available - n ls Output downloaded versions - n ls-remote [version] Output matching versions available for download - n uninstall Remove the installed Node.js - -Options: - - -V, --version Output version of n - -h, --help Display help information - -p, --preserve Preserve npm and npx during install of Node.js - -q, --quiet Disable curl output. Disable log messages processing "auto" and "engine" labels. - -d, --download Download if necessary, and don't make active - -a, --arch Override system architecture - --offline Resolve target version against cached downloads instead of internet lookup - --all ls-remote displays all matches instead of last 20 - --insecure Turn off certificate checking for https requests (may be needed from behind a proxy server) - --use-xz/--no-use-xz Override automatic detection of xz support and enable/disable use of xz compressed node downloads. - -Aliases: - - install: i - latest: current - ls: list - lsr: ls-remote - lts: stable - rm: - - run: use, as - which: bin - -Versions: - - Numeric version numbers can be complete or incomplete, with an optional leading 'v'. - Versions can also be specified by label, or codename, - and other downloadable releases by / - - 4.9.1, 8, v6.1 Numeric versions - lts Newest Long Term Support official release - latest, current Newest official release - auto Read version from file: .n-node-version, .node-version, .nvmrc, or package.json - engine Read version from package.json - boron, carbon Codenames for release streams - lts_latest Node.js support aliases - - and nightly, rc/10 et al - -EOF -} - -err_no_installed_print_help() { - display_help - abort "no downloaded versions yet, see above help for commands" -} - -# -# Synopsis: next_version_installed selected_version -# Output version after selected (which may be blank under some circumstances). -# - -function next_version_installed() { - display_cache_versions | n_grep "$1" -A 1 | tail -n 1 -} - -# -# Synopsis: prev_version_installed selected_version -# Output version before selected (which may be blank under some circumstances). -# - -function prev_version_installed() { - display_cache_versions | n_grep "$1" -B 1 | head -n 1 -} - -# -# Output n version. -# - -display_n_version() { - echo "$VERSION" && exit 0 -} - -# -# Synopsis: set_active_node -# Checks cached downloads for a binary matching the active node. -# Globals modified: -# - g_active_node -# - -function set_active_node() { - g_active_node= - local node_path="$(command -v node)" - if [[ -x "${node_path}" ]]; then - local installed_version=$(node --version) - installed_version=${installed_version#v} - for dir in "${CACHE_DIR}"/*/ ; do - local folder_name="${dir%/}" - folder_name="${folder_name##*/}" - if diff &> /dev/null \ - "${CACHE_DIR}/${folder_name}/${installed_version}/bin/node" \ - "${node_path}" ; then - g_active_node="${folder_name}/${installed_version}" - break - fi - done - fi -} - -# -# Display sorted versions directories paths. -# - -display_versions_paths() { - find "$CACHE_DIR" -maxdepth 2 -type d \ - | sed 's|'"$CACHE_DIR"'/||g' \ - | n_grep -E "/[0-9]+\.[0-9]+\.[0-9]+" \ - | sed 's|/|.|' \ - | sort -k 1,1 -k 2,2n -k 3,3n -k 4,4n -t . \ - | sed 's|\.|/|' -} - -# -# Display installed versions with -# - -display_versions_with_selected() { - local selected="$1" - echo - for version in $(display_versions_paths); do - if test "$version" = "$selected"; then - printf " ${SGR_CYAN}ο${SGR_RESET} %s\n" "$version" - else - printf " ${SGR_FAINT}%s${SGR_RESET}\n" "$version" - fi - done - echo - printf "Use up/down arrow keys to select a version, return key to install, d to delete, q to quit" -} - -# -# Synopsis: display_cache_versions -# - -function display_cache_versions() { - for folder_and_version in $(display_versions_paths); do - echo "${folder_and_version}" - done -} - -# -# Display current node --version and others installed. -# - -menu_select_cache_versions() { - enter_fullscreen - set_active_node - local selected="${g_active_node}" - - clear - display_versions_with_selected "${selected}" - - trap handle_sigint INT - trap handle_sigtstp SIGTSTP - - ESCAPE_SEQ=$'\033' - UP=$'A' - DOWN=$'B' - CTRL_P=$'\020' - CTRL_N=$'\016' - - while true; do - read -rsn 1 key - case "$key" in - "$ESCAPE_SEQ") - # Handle ESC sequences followed by other characters, i.e. arrow keys - read -rsn 1 -t 1 tmp - # See "[" if terminal in normal mode, and "0" in application mode - if [[ "$tmp" == "[" || "$tmp" == "O" ]]; then - read -rsn 1 -t 1 arrow - case "$arrow" in - "$UP") - clear - selected="$(prev_version_installed "${selected}")" - display_versions_with_selected "${selected}" - ;; - "$DOWN") - clear - selected="$(next_version_installed "${selected}")" - display_versions_with_selected "${selected}" - ;; - esac - fi - ;; - "d") - if [[ -n "${selected}" ]]; then - clear - # Note: prev/next is constrained to min/max - local after_delete_selection="$(next_version_installed "${selected}")" - if [[ "${after_delete_selection}" == "${selected}" ]]; then - after_delete_selection="$(prev_version_installed "${selected}")" - fi - remove_versions "${selected}" - - if [[ "${after_delete_selection}" == "${selected}" ]]; then - clear - leave_fullscreen - echo "All downloaded versions have been deleted from cache." - exit - fi - - selected="${after_delete_selection}" - display_versions_with_selected "${selected}" - fi - ;; - # Vim or Emacs 'up' key - "k"|"$CTRL_P") - clear - selected="$(prev_version_installed "${selected}")" - display_versions_with_selected "${selected}" - ;; - # Vim or Emacs 'down' key - "j"|"$CTRL_N") - clear - selected="$(next_version_installed "${selected}")" - display_versions_with_selected "${selected}" - ;; - "q") - clear - leave_fullscreen - exit - ;; - "") - # enter key returns empty string - leave_fullscreen - [[ -n "${selected}" ]] && activate "${selected}" - exit - ;; - esac - done -} - -# -# Move up a line and erase. -# - -erase_line() { - printf "\033[1A\033[2K" -} - -# -# Disable PaX mprotect for -# - -disable_pax_mprotect() { - test -z "$1" && abort "binary required" - local binary="$1" - - # try to disable mprotect via XATTR_PAX header - local PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl-ng 2>&1)" - local PAXCTL_ERROR=1 - if [ -x "$PAXCTL" ]; then - $PAXCTL -l && $PAXCTL -m "$binary" >/dev/null 2>&1 - PAXCTL_ERROR="$?" - fi - - # try to disable mprotect via PT_PAX header - if [ "$PAXCTL_ERROR" != 0 ]; then - PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl 2>&1)" - if [ -x "$PAXCTL" ]; then - $PAXCTL -Cm "$binary" >/dev/null 2>&1 - fi - fi -} - -# -# clean_copy_folder -# - -clean_copy_folder() { - local source="$1" - local target="$2" - if [[ -d "${source}" ]]; then - rm -rf "${target}" - cp -fR "${source}" "${target}" - fi -} - -# -# Activate -# - -activate() { - local version="$1" - local dir="$CACHE_DIR/$version" - local original_node="$(command -v node)" - local installed_node="${N_PREFIX}/bin/node" - log "copying" "$version" - - - # Ideally we would just copy from cache to N_PREFIX, but there are some complications - # - various linux versions use symlinks for folders in /usr/local and also error when copy folder onto symlink - # - we have used cp for years, so keep using it for backwards compatibility (instead of say rsync) - # - we allow preserving npm - # - we want to be somewhat robust to changes in tarball contents, so use find instead of hard-code expected subfolders - # - # This code was purist and concise for a long time. - # Now twice as much code, but using same code path for all uses, and supporting more setups. - - # Copy lib before bin so symlink targets exist. - # lib - mkdir -p "$N_PREFIX/lib" - # Copy everything except node_modules. - find "$dir/lib" -mindepth 1 -maxdepth 1 \! -name node_modules -exec cp -fR "{}" "$N_PREFIX/lib" \; - if [[ -z "${N_PRESERVE_NPM}" ]]; then - mkdir -p "$N_PREFIX/lib/node_modules" - # Copy just npm, skipping possible added global modules after download. Clean copy to avoid version change problems. - clean_copy_folder "$dir/lib/node_modules/npm" "$N_PREFIX/lib/node_modules/npm" - fi - # Takes same steps for corepack (experimental in node 16.9.0) as for npm, to avoid version problems. - if [[ -e "$dir/lib/node_modules/corepack" && -z "${N_PRESERVE_COREPACK}" ]]; then - mkdir -p "$N_PREFIX/lib/node_modules" - clean_copy_folder "$dir/lib/node_modules/corepack" "$N_PREFIX/lib/node_modules/corepack" - fi - - # bin - mkdir -p "$N_PREFIX/bin" - # Remove old node to avoid potential problems with firewall getting confused on Darwin by overwrite. - rm -f "$N_PREFIX/bin/node" - # Copy bin items by hand, in case user has installed global npm modules into cache. - cp -f "$dir/bin/node" "$N_PREFIX/bin" - [[ -e "$dir/bin/node-waf" ]] && cp -f "$dir/bin/node-waf" "$N_PREFIX/bin" # v0.8.x - if [[ -z "${N_PRESERVE_COREPACK}" ]]; then - [[ -e "$dir/bin/corepack" ]] && cp -fR "$dir/bin/corepack" "$N_PREFIX/bin" # from 16.9.0 - fi - if [[ -z "${N_PRESERVE_NPM}" ]]; then - [[ -e "$dir/bin/npm" ]] && cp -fR "$dir/bin/npm" "$N_PREFIX/bin" - [[ -e "$dir/bin/npx" ]] && cp -fR "$dir/bin/npx" "$N_PREFIX/bin" - fi - - # include - mkdir -p "$N_PREFIX/include" - find "$dir/include" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/include" \; - - # share - mkdir -p "$N_PREFIX/share" - # Copy everything except man, at it is a symlink on some Linux (e.g. archlinux). - find "$dir/share" -mindepth 1 -maxdepth 1 \! -name man -exec cp -fR "{}" "$N_PREFIX/share" \; - mkdir -p "$N_PREFIX/share/man" - find "$dir/share/man" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/share/man" \; - - disable_pax_mprotect "${installed_node}" - - local active_node="$(command -v node)" - if [[ -e "${active_node}" && -e "${installed_node}" && "${active_node}" != "${installed_node}" ]]; then - # Installed and active are different which might be a PATH problem. List both to give user some clues. - log "installed" "$("${installed_node}" --version) to ${installed_node}" - log "active" "$("${active_node}" --version) at ${active_node}" - else - local npm_version_str="" - local installed_npm="${N_PREFIX}/bin/npm" - local active_npm="$(command -v npm)" - if [[ -z "${N_PRESERVE_NPM}" && -e "${active_npm}" && -e "${installed_npm}" && "${active_npm}" = "${installed_npm}" ]]; then - npm_version_str=" (with npm $(npm --version))" - fi - - log "installed" "$("${installed_node}" --version)${npm_version_str}" - - # Extra tips for changed location. - if [[ -e "${active_node}" && -e "${original_node}" && "${active_node}" != "${original_node}" ]]; then - printf '\nNote: the node command changed location and the old location may be remembered in your current shell.\n' - log old "${original_node}" - log new "${active_node}" - printf 'If "node --version" shows the old version then start a new shell, or reset the location hash with:\nhash -r (for bash, zsh, ash, dash, and ksh)\nrehash (for csh and tcsh)\n' - fi - fi -} - -# -# Install -# - -install() { - [[ -z "$1" ]] && abort "version required" - local version - get_latest_resolved_version "$1" || return 2 - version="${g_target_node}" - [[ -n "${version}" ]] || abort "no version found for '$1'" - update_mirror_settings_for_version "$1" - update_xz_settings_for_version "${version}" - update_arch_settings_for_version "${version}" - - local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" - - # Note: decompression flags ignored with default Darwin tar which autodetects. - if test "$N_USE_XZ" = "true"; then - local tarflag="-Jx" - else - local tarflag="-zx" - fi - - if test -d "$dir"; then - if [[ ! -e "$dir/n.lock" ]] ; then - if [[ "$DOWNLOAD" == "false" ]] ; then - activate "${g_mirror_folder_name}/${version}" - fi - exit - fi - fi - if [[ "$OFFLINE" == "true" ]]; then - abort "version unavailable offline" - fi - - log installing "${g_mirror_folder_name}-v$version" - - local url="$(tarball_url "$version")" - is_ok "${url}" || abort "download preflight failed for '$version' (${url})" - - log mkdir "$dir" - mkdir -p "$dir" || abort "sudo required (or change ownership, or define N_PREFIX)" - touch "$dir/n.lock" - - cd "${dir}" || abort "Failed to cd to ${dir}" - - log fetch "$url" - do_get "${url}" | tar "$tarflag" --strip-components=1 --no-same-owner -f - - pipe_results=( "${PIPESTATUS[@]}" ) - if [[ "${pipe_results[0]}" -ne 0 ]]; then - abort "failed to download archive for $version" - fi - if [[ "${pipe_results[1]}" -ne 0 ]]; then - abort "failed to extract archive for $version" - fi - [ "$GET_SHOWS_PROGRESS" = "true" ] && erase_line - rm -f "$dir/n.lock" - - disable_pax_mprotect bin/node - - if [[ "$DOWNLOAD" == "false" ]]; then - activate "${g_mirror_folder_name}/$version" - fi -} - -# -# Be more silent. -# - -set_quiet() { - SHOW_VERBOSE_LOG="false" - command -v curl > /dev/null && CURL_OPTIONS+=( "--silent" ) && GET_SHOWS_PROGRESS="false" -} - -# -# Synopsis: do_get [option...] url -# Call curl or wget with combination of global and passed options. -# - -function do_get() { - if command -v curl &> /dev/null; then - curl "${CURL_OPTIONS[@]}" "$@" - elif command -v wget &> /dev/null; then - wget "${WGET_OPTIONS[@]}" "$@" - else - abort "curl or wget command required" - fi -} - -# -# Synopsis: do_get_index [option...] url -# Call curl or wget with combination of global and passed options, -# with options tweaked to be more suitable for getting index. -# - -function do_get_index() { - if command -v curl &> /dev/null; then - # --silent to suppress progress et al - curl --silent "${CURL_OPTIONS[@]}" "$@" - elif command -v wget &> /dev/null; then - wget "${WGET_OPTIONS[@]}" "$@" - else - abort "curl or wget command required" - fi -} - -# -# Synopsis: remove_versions version ... -# - -function remove_versions() { - [[ -z "$1" ]] && abort "version(s) required" - while [[ $# -ne 0 ]]; do - local version - get_latest_resolved_version "$1" || break - version="${g_target_node}" - if [[ -n "${version}" ]]; then - update_mirror_settings_for_version "$1" - local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" - if [[ -s "${dir}" ]]; then - rm -rf "${dir}" - else - echo "$1 (${version}) not in downloads cache" - fi - else - echo "No version found for '$1'" - fi - shift - done -} - -# -# Synopsis: prune_cache -# - -function prune_cache() { - set_active_node - - for folder_and_version in $(display_versions_paths); do - if [[ "${folder_and_version}" != "${g_active_node}" ]]; then - echo "${folder_and_version}" - rm -rf "${CACHE_DIR:?}/${folder_and_version}" - fi - done -} - -# -# Synopsis: find_cached_version version -# Finds cache directory for resolved version. -# Globals modified: -# - g_cached_version - -function find_cached_version() { - [[ -z "$1" ]] && abort "version required" - local version - get_latest_resolved_version "$1" || exit 1 - version="${g_target_node}" - [[ -n "${version}" ]] || abort "no version found for '$1'" - - update_mirror_settings_for_version "$1" - g_cached_version="${CACHE_DIR}/${g_mirror_folder_name}/${version}" - if [[ ! -d "${g_cached_version}" && "${DOWNLOAD}" == "true" ]]; then - (install "${version}") - fi - [[ -d "${g_cached_version}" ]] || abort "'$1' (${version}) not in downloads cache" -} - - -# -# Synopsis: display_bin_path_for_version version -# - -function display_bin_path_for_version() { - find_cached_version "$1" - echo "${g_cached_version}/bin/node" -} - -# -# Synopsis: run_with_version version [args...] -# Run the given of node with [args ..] -# - -function run_with_version() { - find_cached_version "$1" - shift # remove version from parameters - exec "${g_cached_version}/bin/node" "$@" -} - -# -# Synopsis: exec_with_version command [args...] -# Modify the path to include and execute command. -# - -function exec_with_version() { - find_cached_version "$1" - shift # remove version from parameters - PATH="${g_cached_version}/bin:$PATH" exec "$@" -} - -# -# Synopsis: is_ok url -# Check the HEAD response of . -# - -function is_ok() { - # Note: both curl and wget can follow redirects, as present on some mirrors (e.g. https://npm.taobao.org/mirrors/node). - # The output is complicated with redirects, so keep it simple and use command status rather than parse output. - if command -v curl &> /dev/null; then - do_get --silent --head "$1" > /dev/null || return 1 - else - do_get --spider "$1" > /dev/null || return 1 - fi -} - -# -# Synopsis: can_use_xz -# Test system to see if xz decompression is supported by tar. -# - -function can_use_xz() { - # Be conservative and only enable if xz is likely to work. Unfortunately we can't directly query tar itself. - # For research, see https://github.com/shadowspawn/nvh/issues/8 - local uname_s="$(uname -s)" - if [[ "${uname_s}" = "Linux" ]] && command -v xz &> /dev/null ; then - # tar on linux is likely to support xz if it is available as a command - return 0 - elif [[ "${uname_s}" = "Darwin" ]]; then - local macos_version="$(sw_vers -productVersion)" - local macos_major_version="$(echo "${macos_version}" | cut -d '.' -f 1)" - local macos_minor_version="$(echo "${macos_version}" | cut -d '.' -f 2)" - if [[ "${macos_major_version}" -gt 10 || "${macos_minor_version}" -gt 8 ]]; then - # tar on recent Darwin has xz support built-in - return 0 - fi - fi - return 2 # not supported -} - -# -# Synopsis: display_tarball_platform -# - -function display_tarball_platform() { - # https://en.wikipedia.org/wiki/Uname - - local os="unexpected_os" - local uname_a="$(uname -a)" - case "${uname_a}" in - Linux*) os="linux" ;; - Darwin*) os="darwin" ;; - SunOS*) os="sunos" ;; - AIX*) os="aix" ;; - CYGWIN*) >&2 echo_red "Cygwin is not supported by n" ;; - MINGW*) >&2 echo_red "Git BASH (MSYS) is not supported by n" ;; - esac - - local arch="unexpected_arch" - local uname_m="$(uname -m)" - case "${uname_m}" in - x86_64) arch=x64 ;; - i386 | i686) arch="x86" ;; - aarch64) arch=arm64 ;; - armv8l) arch=arm64 ;; # armv8l probably supports arm64, and there is no specific armv8l build so give it a go - *) - # e.g. armv6l, armv7l, arm64 - arch="${uname_m}" - ;; - esac - # Override from command line, or version specific adjustment. - [ -n "$ARCH" ] && arch="$ARCH" - - echo "${os}-${arch}" -} - -# -# Synopsis: display_compatible_file_field -# display for current platform, as per field in index.tab, which is different than actual download -# - -function display_compatible_file_field { - local compatible_file_field="$(display_tarball_platform)" - if [[ -z "${ARCH}" && "${compatible_file_field}" = "darwin-arm64" ]]; then - # Look for arm64 for native but also x64 for older versions which can run in rosetta. - # (Downside is will get an install error if install version above 16 with x64 and not arm64.) - compatible_file_field="osx-arm64-tar|osx-x64-tar" - elif [[ "${compatible_file_field}" =~ darwin-(.*) ]]; then - compatible_file_field="osx-${BASH_REMATCH[1]}-tar" - fi - echo "${compatible_file_field}" -} - -# -# Synopsis: tarball_url version -# - -function tarball_url() { - local version="$1" - local ext=gz - [ "$N_USE_XZ" = "true" ] && ext="xz" - echo "${g_mirror_url}/v${version}/node-v${version}-$(display_tarball_platform).tar.${ext}" -} - -# -# Synopsis: get_file_node_version filename -# Sets g_target_node -# - -function get_file_node_version() { - g_target_node= - local filepath="$1" - verbose_log "found" "${filepath}" - # read returns a non-zero status but does still work if there is no line ending - local version - <"${filepath}" read -r version - # trim possible trailing \d from a Windows created file - version="${version%%[[:space:]]}" - verbose_log "read" "${version}" - g_target_node="${version}" -} - -# -# Synopsis: get_package_engine_version\ -# Sets g_target_node -# - -function get_package_engine_version() { - g_target_node= - local filepath="$1" - verbose_log "found" "${filepath}" - command -v node &> /dev/null || abort "an active version of node is required to read 'engines' from package.json" - local range - range="$(node -e "package = require('${filepath}'); if (package && package.engines && package.engines.node) console.log(package.engines.node)")" - verbose_log "read" "${range}" - [[ -n "${range}" ]] || return 2 - if [[ "*" == "${range}" ]]; then - verbose_log "target" "current" - g_target_node="current" - return - fi - - local version - if [[ "${range}" =~ ^([>~^=]|\>\=)?v?([0-9]+(\.[0-9]+){0,2})(.[xX*])?$ ]]; then - local operator="${BASH_REMATCH[1]}" - version="${BASH_REMATCH[2]}" - case "${operator}" in - '' | =) ;; - \> | \>=) version="current" ;; - \~) [[ "${version}" =~ ^([0-9]+\.[0-9]+)\.[0-9]+$ ]] && version="${BASH_REMATCH[1]}" ;; - ^) [[ "${version}" =~ ^([0-9]+) ]] && version="${BASH_REMATCH[1]}" ;; - esac - verbose_log "target" "${version}" - else - command -v npx &> /dev/null || abort "an active version of npx is required to use complex 'engine' ranges from package.json" - [[ "$OFFLINE" != "true" ]] || abort "offline: an internet connection is required for looking up complex 'engine' ranges from package.json" - verbose_log "resolving" "${range}" - local version_per_line="$(n lsr --all)" - local versions_one_line=$(echo "${version_per_line}" | tr '\n' ' ') - # Using semver@7 so works with older versions of node. - # shellcheck disable=SC2086 - version=$(npm_config_yes=true npx --quiet semver@7 -r "${range}" ${versions_one_line} | tail -n 1) - fi - g_target_node="${version}" -} - -# -# Synopsis: get_nvmrc_version -# Sets g_target_node -# - -function get_nvmrc_version() { - g_target_node= - local filepath="$1" - verbose_log "found" "${filepath}" - local version - <"${filepath}" read -r version - verbose_log "read" "${version}" - # Translate from nvm aliases - case "${version}" in - lts/\*) version="lts" ;; - lts/*) version="${version:4}" ;; - node) version="current" ;; - *) ;; - esac - g_target_node="${version}" -} - -# -# Synopsis: get_engine_version [error-message] -# Sets g_target_node -# - -function get_engine_version() { - g_target_node= - local error_message="${1-package.json not found}" - local parent - parent="${PWD}" - while [[ -n "${parent}" ]]; do - if [[ -e "${parent}/package.json" ]]; then - get_package_engine_version "${parent}/package.json" - else - parent=${parent%/*} - continue - fi - break - done - [[ -n "${parent}" ]] || abort "${error_message}" - [[ -n "${g_target_node}" ]] || abort "did not find supported version of node in 'engines' field of package.json" -} - -# -# Synopsis: get_auto_version -# Sets g_target_node -# - -function get_auto_version() { - g_target_node= - # Search for a version control file first - local parent - parent="${PWD}" - while [[ -n "${parent}" ]]; do - if [[ -e "${parent}/.n-node-version" ]]; then - get_file_node_version "${parent}/.n-node-version" - elif [[ -e "${parent}/.node-version" ]]; then - get_file_node_version "${parent}/.node-version" - elif [[ -e "${parent}/.nvmrc" ]]; then - get_nvmrc_version "${parent}/.nvmrc" - else - parent=${parent%/*} - continue - fi - break - done - # Fallback to package.json - [[ -n "${parent}" ]] || get_engine_version "no file found for auto version (.n-node-version, .node-version, .nvmrc, or package.json)" - [[ -n "${g_target_node}" ]] || abort "file found for auto did not contain target version of node" -} - -# -# Synopsis: get_latest_resolved_version version -# Sets g_target_node -# - -function get_latest_resolved_version() { - g_target_node= - local version=${1} - simple_version=${version#node/} # Only place supporting node/ [sic] - if is_exact_numeric_version "${simple_version}"; then - # Just numbers, already resolved, no need to lookup first. - simple_version="${simple_version#v}" - g_target_node="${simple_version}" - elif [[ "$OFFLINE" == "true" ]]; then - g_target_node=$(display_local_versions "${version}") - else - # Complicated recognising exact version, KISS and lookup. - g_target_node=$(N_MAX_REMOTE_MATCHES=1 display_remote_versions "$version") - fi -} - -# -# Synopsis: display_remote_index -# index.tab reference: https://github.com/nodejs/nodejs-dist-indexer -# Index fields are: version date files npm v8 uv zlib openssl modules lts security -# KISS and just return fields we currently care about: version files lts -# - -display_remote_index() { - local index_url="${g_mirror_url}/index.tab" - # tail to remove header line - do_get_index "${index_url}" | tail -n +2 | cut -f 1,3,10 - if [[ "${PIPESTATUS[0]}" -ne 0 ]]; then - # Reminder: abort will only exit subshell, but consistent error display - abort "failed to download version index (${index_url})" - fi -} - -# -# Synopsis: display_match_limit limit -# - -function display_match_limit(){ - if [[ "$1" -gt 1 && "$1" -lt 32000 ]]; then - echo "Listing remote... Displaying $1 matches (use --all to see all)." - fi -} - -# -# Synopsis: display_local_versions version -# - -function display_local_versions() { - local version="$1" - local match='.' - verbose_log "offline" "matching cached versions" - - # Transform some labels before processing further. - if is_node_support_version "${version}"; then - version="$(display_latest_node_support_alias "${version}")" - match_count=1 - elif [[ "${version}" = "auto" ]]; then - # suppress stdout logging so lsr layout same as usual for scripting - get_auto_version || return 2 - version="${g_target_node}" - elif [[ "${version}" = "engine" ]]; then - # suppress stdout logging so lsr layout same as usual for scripting - get_engine_version || return 2 - version="${g_target_node}" - fi - - if [[ "${version}" = "latest" || "${version}" = "current" ]]; then - match='^node/.' - elif is_exact_numeric_version "${version}"; then - # Quote any dots in version so they are literal for expression - match="^node/${version//\./\.}" - elif is_numeric_version "${version}"; then - version="${version#v}" - # Quote any dots in version so they are literal for expression - match="${version//\./\.}" - # Avoid 1.2 matching 1.23 - match="^node/${match}[^0-9]" - # elif is_lts_codename "${version}"; then - # see if demand - elif is_download_folder "${version}"; then - match="^${version}/" - # elif is_download_version "${version}"; then - # see if demand - else - abort "invalid version '$1' for offline matching" - fi - - display_versions_paths \ - | n_grep -E "${match}" \ - | tail -n 1 \ - | sed 's|node/||' -} - -# -# Synopsis: display_remote_versions version -# - -function display_remote_versions() { - local version="$1" - update_mirror_settings_for_version "${version}" - local match='.' - local match_count="${N_MAX_REMOTE_MATCHES}" - - # Transform some labels before processing further. - if is_node_support_version "${version}"; then - version="$(display_latest_node_support_alias "${version}")" - match_count=1 - elif [[ "${version}" = "auto" ]]; then - # suppress stdout logging so lsr layout same as usual for scripting - get_auto_version || return 2 - version="${g_target_node}" - elif [[ "${version}" = "engine" ]]; then - # suppress stdout logging so lsr layout same as usual for scripting - get_engine_version || return 2 - version="${g_target_node}" - fi - - if [[ -z "${version}" ]]; then - match='.' - elif [[ "${version}" = "lts" || "${version}" = "stable" ]]; then - match_count=1 - # Codename is last field, first one with a name is newest lts - match="${TAB_CHAR}[a-zA-Z]+\$" - elif [[ "${version}" = "latest" || "${version}" = "current" ]]; then - match_count=1 - match='.' - elif is_numeric_version "${version}"; then - version="v${version#v}" - # Avoid restriction message if exact version - is_exact_numeric_version "${version}" && match_count=1 - # Quote any dots in version so they are literal for expression - match="${version//\./\.}" - # Avoid 1.2 matching 1.23 - match="^${match}[^0-9]" - elif is_lts_codename "${version}"; then - # Capitalise (could alternatively make grep case insensitive) - codename="$(echo "${version:0:1}" | tr '[:lower:]' '[:upper:]')${version:1}" - # Codename is last field - match="${TAB_CHAR}${codename}\$" - elif is_download_folder "${version}"; then - match='.' - elif is_download_version "${version}"; then - version="${version#"${g_mirror_folder_name}"/}" - if [[ "${version}" = "latest" || "${version}" = "current" ]]; then - match_count=1 - match='.' - else - version="v${version#v}" - match="${version//\./\.}" - match="^${match}" # prefix - if is_numeric_version "${version}"; then - # Exact numeric match - match="${match}[^0-9]" - fi - fi - else - abort "invalid version '$1'" - fi - display_match_limit "${match_count}" - - # Implementation notes: - # - using awk rather than head so do not close pipe early on curl - # - restrict search to compatible files as not always available, or not at same time - # - return status of curl command (i.e. PIPESTATUS[0]) - display_remote_index \ - | n_grep -E "$(display_compatible_file_field)" \ - | n_grep -E "${match}" \ - | awk "NR<=${match_count}" \ - | cut -f 1 \ - | n_grep -E -o '[^v].*' - return "${PIPESTATUS[0]}" -} - -# -# Synopsis: delete_with_echo target -# - -function delete_with_echo() { - if [[ -e "$1" ]]; then - echo "$1" - rm -rf "$1" - fi -} - -# -# Synopsis: uninstall_installed -# Uninstall the installed node and npm (leaving alone the cache), -# so undo install, and may expose possible system installed versions. -# - -uninstall_installed() { - # npm: https://docs.npmjs.com/misc/removing-npm - # rm -rf /usr/local/{lib/node{,/.npm,_modules},bin,share/man}/npm* - # node: https://stackabuse.com/how-to-uninstall-node-js-from-mac-osx/ - # Doing it by hand rather than scanning cache, so still works if cache deleted first. - # This covers tarballs for at least node 4 through 10. - - while true; do - read -r -p "Do you wish to delete node and npm from ${N_PREFIX}? " yn - case $yn in - [Yy]* ) break ;; - [Nn]* ) exit ;; - * ) echo "Please answer yes or no.";; - esac - done - - echo "" - echo "Uninstalling node and npm" - delete_with_echo "${N_PREFIX}/bin/node" - delete_with_echo "${N_PREFIX}/bin/npm" - delete_with_echo "${N_PREFIX}/bin/npx" - delete_with_echo "${N_PREFIX}/bin/corepack" - delete_with_echo "${N_PREFIX}/include/node" - delete_with_echo "${N_PREFIX}/lib/dtrace/node.d" - delete_with_echo "${N_PREFIX}/lib/node_modules/npm" - delete_with_echo "${N_PREFIX}/lib/node_modules/corepack" - delete_with_echo "${N_PREFIX}/share/doc/node" - delete_with_echo "${N_PREFIX}/share/man/man1/node.1" - delete_with_echo "${N_PREFIX}/share/systemtap/tapset/node.stp" -} - -# -# Synopsis: show_permission_suggestions -# - -function show_permission_suggestions() { - echo "Suggestions:" - echo "- run n with sudo, or" - if [[ "${N_CACHE_PREFIX}" == "${N_PREFIX}" ]]; then - echo "- define N_PREFIX to a writeable location, or" - else - echo "- define N_PREFIX and N_CACHE_PREFIX to writeable locations, or" - fi -} - -# -# Synopsis: show_diagnostics -# Show environment and check for common problems. -# - -function show_diagnostics() { - echo "This information is to help you diagnose issues, and useful when reporting an issue." - echo "Note: some output may contain passwords. Redact before sharing." - - printf "\n\nCOMMAND LOCATIONS AND VERSIONS\n" - - printf "\nbash\n" - command -v bash && bash --version - - printf "\nn\n" - command -v n && n --version - - printf "\nnode\n" - if command -v node &> /dev/null; then - command -v node && node --version - node -e 'if (process.versions.v8) console.log("JavaScript engine: v8");' - - printf "\nnpm\n" - command -v npm && npm --version - fi - - printf "\ntar\n" - if command -v tar &> /dev/null; then - command -v tar && tar --version - else - echo_red "tar not found. Needed for extracting downloads." - fi - - printf "\ncurl or wget\n" - if command -v curl &> /dev/null; then - command -v curl && curl --version - elif command -v wget &> /dev/null; then - command -v wget && wget --version - else - echo_red "Neither curl nor wget found. Need one of them for downloads." - fi - - printf "\nuname\n" - uname -a - - printf "\n\nSETTINGS\n" - - printf "\nn\n" - echo "node mirror: ${N_NODE_MIRROR}" - echo "node downloads mirror: ${N_NODE_DOWNLOAD_MIRROR}" - echo "install destination: ${N_PREFIX}" - [[ -n "${N_PREFIX}" ]] && echo "PATH: ${PATH}" - echo "ls-remote max matches: ${N_MAX_REMOTE_MATCHES}" - [[ -n "${N_PRESERVE_NPM}" ]] && echo "installs preserve npm by default" - [[ -n "${N_PRESERVE_COREPACK}" ]] && echo "installs preserve corepack by default" - - printf "\nProxy\n" - # disable "var is referenced but not assigned": https://github.com/koalaman/shellcheck/wiki/SC2154 - # shellcheck disable=SC2154 - [[ -n "${http_proxy}" ]] && echo "http_proxy: ${http_proxy}" - # shellcheck disable=SC2154 - [[ -n "${https_proxy}" ]] && echo "https_proxy: ${https_proxy}" - if command -v curl &> /dev/null; then - # curl supports lower case and upper case! - # shellcheck disable=SC2154 - [[ -n "${all_proxy}" ]] && echo "all_proxy: ${all_proxy}" - [[ -n "${ALL_PROXY}" ]] && echo "ALL_PROXY: ${ALL_PROXY}" - [[ -n "${HTTP_PROXY}" ]] && echo "HTTP_PROXY: ${HTTP_PROXY}" - [[ -n "${HTTPS_PROXY}" ]] && echo "HTTPS_PROXY: ${HTTPS_PROXY}" - if [[ -e "${CURL_HOME}/.curlrc" ]]; then - echo "have \$CURL_HOME/.curlrc" - elif [[ -e "${HOME}/.curlrc" ]]; then - echo "have \$HOME/.curlrc" - fi - elif command -v wget &> /dev/null; then - if [[ -e "${WGETRC}" ]]; then - echo "have \$WGETRC" - elif [[ -e "${HOME}/.wgetrc" ]]; then - echo "have \$HOME/.wgetrc" - fi - fi - - printf "\n\nCHECKS\n" - - printf "\nChecking n install destination is in PATH...\n" - local install_bin="${N_PREFIX}/bin" - local path_wth_guards=":${PATH}:" - if [[ "${path_wth_guards}" =~ :${install_bin}/?: ]]; then - printf "good\n" - else - echo_red "'${install_bin}' is not in PATH" - fi - if command -v node &> /dev/null; then - printf "\nChecking n install destination priority in PATH...\n" - local node_dir="$(dirname "$(command -v node)")" - - local index=0 - local path_entry - local path_entries - local install_bin_index=0 - local node_index=999 - IFS=':' read -ra path_entries <<< "${PATH}" - for path_entry in "${path_entries[@]}"; do - (( index++ )) - [[ "${path_entry}" =~ ^${node_dir}/?$ ]] && node_index="${index}" - [[ "${path_entry}" =~ ^${install_bin}/?$ ]] && install_bin_index="${index}" - done - if [[ "${node_index}" -lt "${install_bin_index}" ]]; then - echo_red "There is a version of node installed which will be found in PATH before the n installed version." - else - printf "good\n" - fi - fi - - # Check npm too. Simpler check than for PATH and node, more like the runtime logging for active/installed node. - if [[ -z "${N_PRESERVE_NPM}" ]]; then - printf "\nChecking npm install destination...\n" - local installed_npm="${N_PREFIX}/bin/npm" - local active_npm="$(command -v npm)" - if [[ -e "${active_npm}" && -e "${installed_npm}" && "${active_npm}" != "${installed_npm}" ]]; then - echo_red "There is an active version of npm shadowing the version installed by n. Check order of entries in PATH." - log "installed" "${installed_npm}" - log "active" "${active_npm}" - else - printf "good\n" - fi - fi - - printf "\nChecking prefix folders...\n" - if [[ ! -e "${N_PREFIX}" ]]; then - echo "Folder does not exist: ${N_PREFIX}" - echo "- This folder will be created when you do an install." - fi - if [[ "${N_PREFIX}" != "${N_CACHE_PREFIX}" && ! -e "${N_CACHE_PREFIX}" ]]; then - echo "Folder does not exist: ${N_CACHE_PREFIX}" - echo "- This folder will be created when you do an install." - fi - if [[ -e "${N_PREFIX}" && -e "${N_CACHE_PREFIX}" ]]; then - echo "good" - fi - - if [[ -e "${N_CACHE_PREFIX}" ]]; then - printf "\nChecking permissions for cache folder...\n" - # Using knowledge cache path ends in /n/versions in following check. - if [[ ! -e "${CACHE_DIR}" && (( -e "${N_CACHE_PREFIX}/n" && ! -w "${N_CACHE_PREFIX}/n" ) || ( ! -e "${N_CACHE_PREFIX}/n" && ! -w "${N_CACHE_PREFIX}" )) ]]; then - echo_red "You do not have write permission to create: ${CACHE_DIR}" - show_permission_suggestions - echo "- make a folder you own:" - echo " sudo mkdir -p \"${CACHE_DIR}\"" - echo " sudo chown $(whoami) \"${CACHE_DIR}\"" - elif [[ ! -e "${CACHE_DIR}" ]]; then - echo "Cache folder does not exist: ${CACHE_DIR}" - echo "- This is normal if you have not done an install yet, as cache is only created when needed." - elif [[ ! -w "${CACHE_DIR}" ]]; then - echo_red "You do not have write permission to: ${CACHE_DIR}" - show_permission_suggestions - echo "- change folder ownership to yourself:" - echo " sudo chown -R $(whoami) \"${CACHE_DIR}\"" - else - echo "good" - fi - fi - - if [[ -e "${N_PREFIX}" ]]; then - printf "\nChecking permissions for install folders...\n" - local install_writeable="true" - for subdir in bin lib include share; do - if [[ -e "${N_PREFIX}/${subdir}" && ! -w "${N_PREFIX}/${subdir}" ]]; then - install_writeable="false" - echo_red "You do not have write permission to: ${N_PREFIX}/${subdir}" - break - fi - if [[ ! -e "${N_PREFIX}/${subdir}" && ! -w "${N_PREFIX}" ]]; then - install_writeable="false" - echo_red "You do not have write permission to create: ${N_PREFIX}/${subdir}" - break - fi - done - if [[ "${install_writeable}" = "true" ]]; then - echo "good" - else - show_permission_suggestions - echo "- change folder ownerships to yourself:" - echo " cd \"${N_PREFIX}\"" - echo " sudo mkdir -p bin lib include share" - echo " sudo chown -R $(whoami) bin lib include share" - fi - fi - - printf "\nChecking mirror is reachable...\n" - if is_ok "${N_NODE_MIRROR}/"; then - printf "good\n" - else - echo_red "mirror not reachable" - printf "Showing failing command and output\n" - if command -v curl &> /dev/null; then - ( set -x; do_get --head "${N_NODE_MIRROR}/" ) - else - ( set -x; do_get --spider "${N_NODE_MIRROR}/" ) - printf "\n" - fi - fi -} - -# -# Handle arguments. -# - -# First pass. Process the options so they can come before or after commands, -# particularly for `n lsr --all` and `n install --arch x686` -# which feel pretty natural. - -unprocessed_args=() -positional_arg="false" - -while [[ $# -ne 0 ]]; do - case "$1" in - --all) N_MAX_REMOTE_MATCHES=32000 ;; - -V|--version) display_n_version ;; - -h|--help|help) display_help; exit ;; - -q|--quiet) set_quiet ;; - -d|--download) DOWNLOAD="true" ;; - --offline) OFFLINE="true" ;; - --insecure) set_insecure ;; - -p|--preserve) N_PRESERVE_NPM="true" N_PRESERVE_COREPACK="true" ;; - --no-preserve) N_PRESERVE_NPM="" N_PRESERVE_COREPACK="" ;; - --use-xz) N_USE_XZ="true" ;; - --no-use-xz) N_USE_XZ="false" ;; - --latest) display_remote_versions latest; exit ;; - --stable) display_remote_versions lts; exit ;; # [sic] old terminology - --lts) display_remote_versions lts; exit ;; - -a|--arch) shift; set_arch "$1";; # set arch and continue - exec|run|as|use) - unprocessed_args+=( "$1" ) - positional_arg="true" - ;; - *) - if [[ "${positional_arg}" == "true" ]]; then - unprocessed_args+=( "$@" ) - break - fi - unprocessed_args+=( "$1" ) - ;; - esac - shift -done - -if [[ -z "${N_USE_XZ+defined}" ]]; then - N_USE_XZ="true" # Default to using xz - can_use_xz || N_USE_XZ="false" -fi - -set -- "${unprocessed_args[@]}" - -if test $# -eq 0; then - test -z "$(display_versions_paths)" && err_no_installed_print_help - menu_select_cache_versions -else - while test $# -ne 0; do - case "$1" in - bin|which) display_bin_path_for_version "$2"; exit ;; - run|as|use) shift; run_with_version "$@"; exit ;; - exec) shift; exec_with_version "$@"; exit ;; - doctor) show_diagnostics; exit ;; - rm|-) shift; remove_versions "$@"; exit ;; - prune) prune_cache; exit ;; - latest) install latest; exit ;; - stable) install stable; exit ;; - lts) install lts; exit ;; - ls|list) display_versions_paths; exit ;; - lsr|ls-remote|list-remote) shift; display_remote_versions "$1"; exit ;; - uninstall) uninstall_installed; exit ;; - i|install) shift; install "$1"; exit ;; - N_TEST_DISPLAY_LATEST_RESOLVED_VERSION) shift; get_latest_resolved_version "$1" > /dev/null || exit 2; echo "${g_target_node}"; exit ;; - *) install "$1"; exit ;; - esac - shift - done -fi From 701828bf4589ed29e797e6b5ebc4bb21b7af4299 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 13:17:26 +0200 Subject: [PATCH 029/146] helpers2.1: simplify backup/restore helper syntax: ynh_restore_file -> ynh_restore to be symetric with ynh_backup. Remove unused --dest_dir arg, rename --src/origin_path to --target --- helpers/helpers.v2.1.d/backup | 134 +++++++++------------------------- 1 file changed, 33 insertions(+), 101 deletions(-) diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 6b147ed6a..21ee091a4 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -4,19 +4,16 @@ CAN_BIND=${CAN_BIND:-1} # Add a file or a directory to the list of paths to backup # -# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory] -# | arg: -s, --src_path= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. -# | arg: -d, --dest_path= - destination file or directory inside the backup dir +# usage: ynh_backup --target=/path/to/stuff [--is_big] [--not_mandatory] +# | arg: -s, --target= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. # | arg: -b, --is_big - Indicate data are big (mail, video, image ...) # | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it. # # This helper can be used both in a system backup hook, and in an app backup script # -# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it +# `ynh_backup` writes `target` and the corresponding path inside the archive (dest_path) into a CSV file, and it # creates the parent destination directory # -# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`. -# # Example in the context of a wordpress app : # ``` # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" @@ -49,7 +46,7 @@ CAN_BIND=${CAN_BIND:-1} # each of his backup. And so handle that part differently. # # As this part of your backup may not be done, your restore script has to handle it. -# In your restore script, use `--not_mandatory` with `ynh_restore_file` +# In your restore script, use `--not_mandatory` with `ynh_restore` # As well in your remove script, you should not remove those data ! Or an user may end up with # a failed upgrade restoring an app without data anymore ! # @@ -65,13 +62,11 @@ ynh_backup() { # TODO find a way to avoid injection by file strange naming ! # ============ Argument parsing ============= - local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory) - local src_path - local dest_path + local -A args_array=([t]=target= [b]=is_big [m]=not_mandatory) + local target local is_big local not_mandatory ynh_handle_getopts_args "$@" - dest_path="${dest_path:-}" is_big="${is_big:-0}" not_mandatory="${not_mandatory:-0}" # =========================================== @@ -83,9 +78,9 @@ ynh_backup() { # don't backup big data items if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then if [ $BACKUP_CORE_ONLY -eq 1 ]; then - ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set." + ynh_print_info --message="$target will not be saved, because 'BACKUP_CORE_ONLY' is set." else - ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set." + ynh_print_info --message="$target will not be saved, because 'do_not_backup_data' is set." fi return 0 fi @@ -94,16 +89,10 @@ ynh_backup() { # Format correctly source and destination paths # ============================================================================== # Be sure the source path is not empty - if [ ! -e "$src_path" ]; then + if [ ! -e "$target" ]; then ynh_print_warn --message="Source path '${src_path}' does not exist" if [ "$not_mandatory" == "0" ]; then - # This is a temporary fix for fail2ban config files missing after the migration to stretch. - if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then - touch "${src_path}" - ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!" - else - return 1 - fi + return 1 else return 0 fi @@ -111,37 +100,11 @@ ynh_backup() { # Transform the source path as an absolute path # If it's a dir remove the ending / - src_path=$(realpath "$src_path") + src_path=$(realpath "$target") - # If there is no destination path, initialize it with the source path - # relative to "/". + # Initialize the dest path with the source path relative to "/". # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost - if [[ -z "$dest_path" ]]; then - dest_path="${src_path#/}" - - else - if [[ "${dest_path:0:1}" == "/" ]]; then - - # If the destination path is an absolute path, transform it as a path - # relative to the current working directory ($YNH_CWD) - # - # If it's an app backup script that run this helper, YNH_CWD is equal to - # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/ - # - # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR - dest_path="${dest_path#$YNH_CWD/}" - - # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD - if [[ "${dest_path:0:1}" == "/" ]]; then - dest_path="${dest_path#/}" - fi - fi - - # Complete dest_path if ended by a / - if [[ "${dest_path: -1}" == "/" ]]; then - dest_path="${dest_path}/$(basename $src_path)" - fi - fi + dest_path="${src_path#/}" # Check if dest_path already exists in tmp archive if [[ -e "${dest_path}" ]]; then @@ -170,25 +133,6 @@ ynh_backup() { mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}") } -# Restore all files that were previously backuped in a core backup script or app backup script -# -# usage: ynh_restore -# -# Requires YunoHost version 2.6.4 or higher. -ynh_restore() { - # Deduce the relative path of $YNH_CWD - local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}" - REL_DIR="${REL_DIR%/}/" - - # For each destination path begining by $REL_DIR - cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \ - | while read line; do - local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)") - local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)") - ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH" - done -} - # Return the path in the archive where has been stocked the origin path # # [internal] @@ -212,17 +156,16 @@ with open(sys.argv[1], 'r') as backup_file: # Restore a file or a directory # -# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory] -# | arg: -o, --origin_path= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive -# | arg: -d, --dest_path= - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv` +# usage: ynh_restore --target=/path/to/stuff [--not_mandatory] +# | arg: -t, --target= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive # | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it. # # Use the registered path in backup_list by ynh_backup to restore the file at the right place. # # examples: -# ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf" +# ynh_restore -t "/etc/nginx/conf.d/$domain.d/$app.conf" # # You can also use relative paths: -# ynh_restore_file -o "conf/nginx.conf" +# ynh_restore -t "conf/nginx.conf" # # If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in # `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed. @@ -234,62 +177,51 @@ with open(sys.argv[1], 'r') as backup_file: # # Requires YunoHost version 2.6.4 or higher. # Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory -ynh_restore_file() { +ynh_restore() { # ============ Argument parsing ============= - local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory) - local origin_path - local dest_path + local -A args_array=([t]=target= [m]=not_mandatory) + local target local not_mandatory ynh_handle_getopts_args "$@" - origin_path="/${origin_path#/}" - # Default value for dest_path = /$origin_path - dest_path="${dest_path:-$origin_path}" + target="/${target#/}" not_mandatory="${not_mandatory:-0}" # =========================================== - local archive_path="$YNH_CWD${origin_path}" + local archive_path="$YNH_CWD${target}" # If archive_path doesn't exist, search for a corresponding path in CSV if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then if [ "$not_mandatory" == "0" ]; then - archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")" + archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$target\")" else return 0 fi fi # Move the old directory if it already exists - if [[ -e "${dest_path}" ]]; then + if [[ -e "${target}" ]]; then # Check if the file/dir size is less than 500 Mo - if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then - local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')" + if [[ $(du --summarize --bytes ${target} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then + local backup_file="/var/cache/yunohost/appconfbackup/${target}.backup.$(date '+%Y%m%d.%H%M%S')" mkdir --parents "$(dirname "$backup_file")" - mv "${dest_path}" "$backup_file" # Move the current file or directory + mv "${target}" "$backup_file" # Move the current file or directory else - ynh_safe_rm --target=${dest_path} + ynh_safe_rm --target=${target} fi fi - # Restore origin_path into dest_path - mkdir --parents $(dirname "$dest_path") + # Restore target into target + mkdir --parents $(dirname "$target") # Do a copy if it's just a mounting point if mountpoint --quiet $YNH_BACKUP_DIR; then if [[ -d "${archive_path}" ]]; then archive_path="${archive_path}/." - mkdir --parents "$dest_path" + mkdir --parents "$target" fi - cp --archive "$archive_path" "${dest_path}" + cp --archive "$archive_path" "${target}" # Do a move if YNH_BACKUP_DIR is already a copy else - mv "$archive_path" "${dest_path}" - fi - - # Boring hack for nginx conf file mapped to php7.3 - # Note that there's no need to patch the fpm config because most php apps - # will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch - if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}" - then - sed -i 's/php7.3/php7.4/g' "${dest_path}" + mv "$archive_path" "${target}" fi } From 0eda746af5c6c8597dea8a4de54c513972401355 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 13:36:00 +0200 Subject: [PATCH 030/146] helpers2.1: simplify ynh_add_fail2ban_config: remove unecessary/unused max_retry and ports options, remove --use_template: just generate the conf on-the-fly if --failregex/--logpath are provided, or use the f2b_stuff templates otherwise --- helpers/helpers.v2.1.d/fail2ban | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index 8e7e34b7a..be9f2da95 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -2,13 +2,11 @@ # Create a dedicated fail2ban config (jail and filter conf files) # -# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] +# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter # | arg: -l, --logpath= - Log file to be checked by fail2ban # | arg: -r, --failregex= - Failregex to be looked for by fail2ban -# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https # -# usage 2: ynh_add_fail2ban_config --use_template +# usage 2: ynh_add_fail2ban_config # | arg: -t, --use_template - Use this helper in template mode # # This will use a template in `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf` @@ -23,7 +21,7 @@ # port = http,https # filter = __APP__ # logpath = /var/log/__APP__/logfile.log -# maxretry = 3 +# maxretry = 5 # ``` # ``` # f2b_filter.conf: @@ -59,30 +57,23 @@ # Requires YunoHost version 4.1.0 or higher. ynh_add_fail2ban_config() { # ============ Argument parsing ============= - local -A args_array=([l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template) + local -A args_array=([l]=logpath= [r]=failregex=) local logpath local failregex - local max_retry - local ports - local use_template ynh_handle_getopts_args "$@" - max_retry=${max_retry:-3} - ports=${ports:-http,https} - use_template="${use_template:-0}" # =========================================== - if [ "$use_template" -ne 1 ]; then - # Usage 1, no template. Build a config file from scratch. + # If failregex is provided, Build a config file on-the-fly using $logpath and $failregex + if [[ -n "${failregex:-}" ]]; then test -n "$logpath" || ynh_die --message="ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die --message="ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." echo " [__APP__] enabled = true -port = __PORTS__ +port = http,https filter = __APP__ logpath = __LOGPATH__ -maxretry = __MAX_RETRY__ +maxretry = 5 " >"$YNH_APP_BASEDIR/conf/f2b_jail.conf" echo " From 67477473e87f77413ef13b71a110eac1ae7061f0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 14:06:49 +0200 Subject: [PATCH 031/146] helpers2.1: remove legacy/unecessary/underused helpers: ynh_print_log, ynh_print_err, ynh_exec_err, ynh_exec_quiet, ynh_exec_fully_quiet, ynh_print_OFF, ynh_print_ON --- helpers/helpers.v2.1.d/apps | 8 +- helpers/helpers.v2.1.d/apt | 2 +- helpers/helpers.v2.1.d/backup | 2 +- helpers/helpers.v2.1.d/fail2ban | 2 +- helpers/helpers.v2.1.d/logging | 137 +----------------------------- helpers/helpers.v2.1.d/postgresql | 2 +- helpers/helpers.v2.1.d/systemd | 8 +- 7 files changed, 13 insertions(+), 148 deletions(-) diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps index c010dab2b..474f5e45f 100644 --- a/helpers/helpers.v2.1.d/apps +++ b/helpers/helpers.v2.1.d/apps @@ -134,7 +134,7 @@ ynh_spawn_app_shell() { # Force Bash to be used to run this helper if [[ ! $0 =~ \/?bash$ ]] then - ynh_print_err --message="Please use Bash as shell" + ynh_print_warn --message="Please use Bash as shell" exit 1 fi @@ -142,13 +142,13 @@ ynh_spawn_app_shell() { local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id)) if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]] then - ynh_print_err --message="$app is not in the apps list" + ynh_print_warn --message="$app is not in the apps list" exit 1 fi # Make sure the app has its own user if ! id -u "$app" &>/dev/null; then - ynh_print_err --message="There is no \"$app\" system user" + ynh_print_warn --message="There is no \"$app\" system user" exit 1 fi @@ -156,7 +156,7 @@ ynh_spawn_app_shell() { local install_dir=$(ynh_app_setting_get --key=install_dir) if [ -z "$install_dir" ] then - ynh_print_err --message="$app has no install_dir setting (does it use packaging format >=2?)" + ynh_print_warn --message="$app has no install_dir setting (does it use packaging format >=2?)" exit 1 fi diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 2e13f1186..24e5b5f99 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -28,7 +28,7 @@ ynh_wait_dpkg_free() { # Check if the name of this file contains only numbers. if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then # If so, that a remaining of dpkg. - ynh_print_err --message="dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + ynh_print_warn --message="dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." set -o xtrace # set -x return 1 fi diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 21ee091a4..0f34e9bc9 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -108,7 +108,7 @@ ynh_backup() { # Check if dest_path already exists in tmp archive if [[ -e "${dest_path}" ]]; then - ynh_print_err --message="Destination path '${dest_path}' already exist" + ynh_print_warn --message="Destination path '${dest_path}' already exist" return 1 fi diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index be9f2da95..05fe0ea2b 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -109,7 +109,7 @@ ignoreregex = local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then - ynh_print_err --message="Fail2ban failed to load the jail for $app" + ynh_print_warn --message="Fail2ban failed to load the jail for $app" ynh_print_warn --message="${fail2ban_error#*WARNING}" fi } diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index 4af30af85..06b8d5083 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -36,15 +36,6 @@ ynh_print_info() { echo "$message" >&$YNH_STDINFO } -# Main printer, just in case in the future we have to change anything about that. -# -# [internal] -# -# Requires YunoHost version 3.2.0 or higher. -ynh_print_log() { - echo -e "${1}" -} - # Print a warning on stderr # # usage: ynh_print_warn --message="Text to print" @@ -58,65 +49,7 @@ ynh_print_warn() { ynh_handle_getopts_args "$@" # =========================================== - ynh_print_log "${message}" >&2 -} - -# Print an error on stderr -# -# usage: ynh_print_err --message="Text to print" -# | arg: -m, --message= - The text to print -# -# Requires YunoHost version 3.2.0 or higher. -ynh_print_err() { - # ============ Argument parsing ============= - local -A args_array=([m]=message=) - local message - ynh_handle_getopts_args "$@" - # =========================================== - - ynh_print_log "[Error] ${message}" >&2 -} - -# Execute a command and print the result as an error -# -# usage: ynh_exec_err your command and args -# | arg: command - command to execute -# -# Note that you should NOT quote the command but only prefix it with ynh_exec_err -# -# Requires YunoHost version 3.2.0 or higher. -ynh_exec_err() { - # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, - # (because in the past eval was used) ... - # we detect this by checking that there's no 2nd arg, and $1 contains a space - if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] - then - ynh_print_err --message="$(eval $@)" - else - # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 - ynh_print_err --message="$("$@")" - fi -} - -# Execute a command and print the result as a warning -# -# usage: ynh_exec_warn your command and args -# | arg: command - command to execute -# -# Note that you should NOT quote the command but only prefix it with ynh_exec_warn -# -# Requires YunoHost version 3.2.0 or higher. -ynh_exec_warn() { - # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, - # (because in the past eval was used) ... - # we detect this by checking that there's no 2nd arg, and $1 contains a space - if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] - then - ynh_print_warn --message="$(eval $@)" - else - # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 - ynh_print_warn --message="$("$@")" - fi + echo -e "${message}" >&2 } # Execute a command and force the result to be printed on stdout @@ -140,48 +73,6 @@ ynh_exec_warn_less() { fi } -# Execute a command and redirect stdout in /dev/null -# -# usage: ynh_exec_quiet your command and args -# | arg: command - command to execute -# -# Note that you should NOT quote the command but only prefix it with ynh_exec_warn -# -# Requires YunoHost version 3.2.0 or higher. -ynh_exec_quiet() { - # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, - # (because in the past eval was used) ... - # we detect this by checking that there's no 2nd arg, and $1 contains a space - if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] - then - eval $@ > /dev/null - else - # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 - "$@" > /dev/null - fi -} - -# Execute a command and redirect stdout and stderr in /dev/null -# -# usage: ynh_exec_quiet your command and args -# | arg: command - command to execute -# -# Note that you should NOT quote the command but only prefix it with ynh_exec_quiet -# -# Requires YunoHost version 3.2.0 or higher. -ynh_exec_fully_quiet() { - # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, - # (because in the past eval was used) ... - # we detect this by checking that there's no 2nd arg, and $1 contains a space - if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] - then - eval $@ > /dev/null 2>&1 - else - # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 - "$@" > /dev/null 2>&1 - fi -} - # Execute a command and redirect stderr in /dev/null. Print stderr on error. # # usage: ynh_exec_and_print_stderr_only_if_error your command and args @@ -202,32 +93,6 @@ ynh_exec_and_print_stderr_only_if_error() { fi } -# Remove any logs for all the following commands. -# -# usage: ynh_print_OFF -# -# [internal] -# -# WARNING: You should be careful with this helper, and never forget to use ynh_print_ON as soon as possible to restore the logging. -# -# Requires YunoHost version 3.2.0 or higher. -ynh_print_OFF() { - exec {BASH_XTRACEFD}>/dev/null -} - -# Restore the logging after ynh_print_OFF -# -# usage: ynh_print_ON -# -# [internal] -# -# Requires YunoHost version 3.2.0 or higher. -ynh_print_ON() { - exec {BASH_XTRACEFD}>&1 - # Print an echo only for the log, to be able to know that ynh_print_ON has been called. - echo ynh_print_ON >/dev/null -} - # Initial definitions for ynh_script_progression increment_progression=0 previous_weight=0 diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 5423026d2..71be36f8f 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -195,7 +195,7 @@ ynh_psql_database_exists() { # though it could exists. if ! command -v psql then - ynh_print_err --message="PostgreSQL is not installed, impossible to check for db existence." + ynh_print_warn --message="PostgreSQL is not installed, impossible to check for db existence." return 1 elif ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then return 1 diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 748c31cfa..8d63006fe 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -112,10 +112,10 @@ ynh_systemd_action() { # If the service fails to perform the action if ! systemctl $action $service_name; then # Show syslog for this service - ynh_exec_err journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name + journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name >&2 # If a log is specified for this service, show also the content of this log if [ -e "$log_path" ]; then - ynh_exec_err tail --lines=$length "$log_path" + tail --lines=$length "$log_path" >&2 fi ynh_clean_check_starting return 1 @@ -160,10 +160,10 @@ ynh_systemd_action() { if [ $i -eq $timeout ]; then ynh_print_warn --message="The service $service_name didn't fully executed the action ${action} before the timeout." ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" - ynh_exec_warn journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name + journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name >&2 if [ -e "$log_path" ]; then ynh_print_warn --message="\-\-\-" - ynh_exec_warn tail --lines=$length "$log_path" + tail --lines=$length "$log_path" >&2 fi fi ynh_clean_check_starting From 0273ee34b171d9d16b78102ca5d119341625b8ed Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 14:11:25 +0200 Subject: [PATCH 032/146] helpers2.1: remove ugly legacy eval trick in ynh_exec_warn_less --- helpers/helpers.v2.1.d/logging | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index 06b8d5083..4721b45dc 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -61,16 +61,8 @@ ynh_print_warn() { # # Requires YunoHost version 3.2.0 or higher. ynh_exec_warn_less() { - # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, - # (because in the past eval was used) ... - # we detect this by checking that there's no 2nd arg, and $1 contains a space - if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] - then - eval $@ 2>&1 - else - # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 - "$@" 2>&1 - fi + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" 2>&1 } # Execute a command and redirect stderr in /dev/null. Print stderr on error. From 9b6ccb7b1f0cf20ebd54b49dfd80f8f7dee63fb8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 14:38:56 +0200 Subject: [PATCH 033/146] helpers2.1: remove a whole bunch of unused args in mongo helpers.. --- helpers/helpers.v2.1.d/mongodb | 300 +++++++++++++-------------------- 1 file changed, 115 insertions(+), 185 deletions(-) diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 608e7f055..1626c27d2 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -5,100 +5,30 @@ # example: ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("wekan")' # example: ynh_mongo_exec --command="db.getMongo().getDBNames().indexOf(\"wekan\")" # -# usage: ynh_mongo_exec [--user=user] [--password=password] [--authenticationdatabase=authenticationdatabase] [--database=database] [--host=host] [--port=port] --command="command" [--eval] -# | arg: -u, --user= - The user name to connect as -# | arg: -p, --password= - The user password -# | arg: -d, --authenticationdatabase= - The authenticationdatabase to connect to -# | arg: -d, --database= - The database to connect to -# | arg: -h, --host= - The host to connect to -# | arg: -P, --port= - The port to connect to -# | arg: -c, --command= - The command to evaluate -# | arg: -e, --eval - Evaluate instead of execute the command. +# usage: ynh_mongo_exec [--database=database] --command="command" +# | arg: -d, --database= - The database to connect to +# | arg: -c, --command= - The command to evaluate # # ynh_mongo_exec() { # ============ Argument parsing ============= - local -A args_array=( [u]=user= [p]=password= [a]=authenticationdatabase= [d]=database= [h]=host= [P]=port= [c]=command= [e]=eval ) - local user - local password - local authenticationdatabase - local database - local host - local port - local command - local eval - ynh_handle_getopts_args "$@" - user="${user:-}" - password="${password:-}" - authenticationdatabase="${authenticationdatabase:-}" - database="${database:-}" - host="${host:-}" - port="${port:-}" - eval=${eval:-0} + local -A args_array=( [d]=database= [c]=command= ) + local database + local command + ynh_handle_getopts_args "$@" + database="${database:-}" # =========================================== - # If user is provided - if [ -n "$user" ] - then - user="--username=$user" - - # If password is provided - if [ -n "$password" ] - then - password="--password=$password" - fi - - # If authenticationdatabase is provided - if [ -n "$authenticationdatabase" ] - then - authenticationdatabase="--authenticationDatabase=$authenticationdatabase" - else - authenticationdatabase="--authenticationDatabase=admin" - fi - else - password="" - authenticationdatabase="" - fi - - # If host is provided - if [ -n "$host" ] - then - host="--host=$host" - fi - - # If port is provided - if [ -n "$port" ] - then - port="--port=$port" - fi - - # If eval is not provided - if [ $eval -eq 0 ] - then - # If database is provided - if [ -n "$database" ] - then - database="use $database" - else - database="" - fi - - mongosh --quiet --username $user --password $password --authenticationDatabase $authenticationdatabase --host $host --port $port < ./dump.bson # # usage: ynh_mongo_dump_db --database=database -# | arg: -d, --database= - The database name to dump +# | arg: -d, --database= - The database name to dump # | ret: the mongodump output # # ynh_mongo_dump_db() { # ============ Argument parsing ============= - local -A args_array=( [d]=database= ) - local database - ynh_handle_getopts_args "$@" + local -A args_array=( [d]=database= ) + local database + ynh_handle_getopts_args "$@" # =========================================== - mongodump --quiet --db="$database" --archive + mongodump --quiet --db="$database" --archive } # Create a user @@ -146,47 +76,47 @@ ynh_mongo_dump_db() { # [internal] # # usage: ynh_mongo_create_user --db_user=user --db_pwd=pwd --db_name=name -# | arg: -u, --db_user= - The user name to create -# | arg: -p, --db_pwd= - The password to identify user by -# | arg: -n, --db_name= - Name of the database to grant privilegies +# | arg: -u, --db_user= - The user name to create +# | arg: -p, --db_pwd= - The password to identify user by +# | arg: -n, --db_name= - Name of the database to grant privilegies # # ynh_mongo_create_user() { # ============ Argument parsing ============= - local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) - local db_user - local db_name - local db_pwd - ynh_handle_getopts_args "$@" + local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) + local db_user + local db_name + local db_pwd + ynh_handle_getopts_args "$@" # =========================================== - # Create the user and set the user as admin of the db - ynh_mongo_exec --database="$db_name" --command='db.createUser( { user: "'${db_user}'", pwd: "'${db_pwd}'", roles: [ { role: "readWrite", db: "'${db_name}'" } ] } );' + # Create the user and set the user as admin of the db + ynh_mongo_exec --database="$db_name" --command='db.createUser( { user: "'${db_user}'", pwd: "'${db_pwd}'", roles: [ { role: "readWrite", db: "'${db_name}'" } ] } );' - # Add clustermonitoring rights - ynh_mongo_exec --database="$db_name" --command='db.grantRolesToUser("'${db_user}'",[{ role: "clusterMonitor", db: "admin" }]);' + # Add clustermonitoring rights + ynh_mongo_exec --database="$db_name" --command='db.grantRolesToUser("'${db_user}'",[{ role: "clusterMonitor", db: "admin" }]);' } # Check if a mongo database exists # # usage: ynh_mongo_database_exists --database=database -# | arg: -d, --database= - The database for which to check existence +# | arg: -d, --database= - The database for which to check existence # | exit: Return 1 if the database doesn't exist, 0 otherwise # # ynh_mongo_database_exists() { # ============ Argument parsing ============= - local -A args_array=([d]=database=) - local database - ynh_handle_getopts_args "$@" + local -A args_array=([d]=database=) + local database + ynh_handle_getopts_args "$@" # =========================================== - if [ $(ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("'${database}'")' --eval) -lt 0 ] - then - return 1 - else - return 0 - fi + if [ $(ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("'${database}'")') -lt 0 ] + then + return 1 + else + return 0 + fi } # Restore a database @@ -194,17 +124,17 @@ ynh_mongo_database_exists() { # example: ynh_mongo_restore_db --database=wekan < ./dump.bson # # usage: ynh_mongo_restore_db --database=database -# | arg: -d, --database= - The database name to restore +# | arg: -d, --database= - The database name to restore # # ynh_mongo_restore_db() { # ============ Argument parsing ============= - local -A args_array=( [d]=database= ) - local database - ynh_handle_getopts_args "$@" + local -A args_array=( [d]=database= ) + local database + ynh_handle_getopts_args "$@" # =========================================== - mongorestore --quiet --db="$database" --archive + mongorestore --quiet --db="$database" --archive } # Drop a user @@ -212,27 +142,27 @@ ynh_mongo_restore_db() { # [internal] # # usage: ynh_mongo_drop_user --db_user=user --db_name=name -# | arg: -u, --db_user= - The user to drop -# | arg: -n, --db_name= - Name of the database +# | arg: -u, --db_user= - The user to drop +# | arg: -n, --db_name= - Name of the database # # ynh_mongo_drop_user() { # ============ Argument parsing ============= - local -A args_array=( [u]=db_user= [n]=db_name= ) - local db_user - local db_name - ynh_handle_getopts_args "$@" + local -A args_array=( [u]=db_user= [n]=db_name= ) + local db_user + local db_name + ynh_handle_getopts_args "$@" # =========================================== - ynh_mongo_exec --database="$db_name" --command='db.dropUser("'$db_user'", {w: "majority", wtimeout: 5000})' + ynh_mongo_exec --database="$db_name" --command='db.dropUser("'$db_user'", {w: "majority", wtimeout: 5000})' } # Create a database, an user and its password. Then store the password in the app's config # # usage: ynh_mongo_setup_db --db_user=user --db_name=name [--db_pwd=pwd] -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database -# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated # # After executing this helper, the password of the created database will be available in $db_pwd # It will also be stored as "mongopwd" into the app settings. @@ -240,67 +170,67 @@ ynh_mongo_drop_user() { # ynh_mongo_setup_db() { # ============ Argument parsing ============= - local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) - local db_user - local db_name - db_pwd="" - ynh_handle_getopts_args "$@" + local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) + local db_user + local db_name + db_pwd="" + ynh_handle_getopts_args "$@" # =========================================== - local new_db_pwd=$(ynh_string_random) # Generate a random password - # If $db_pwd is not provided, use new_db_pwd instead for db_pwd - db_pwd="${db_pwd:-$new_db_pwd}" + local new_db_pwd=$(ynh_string_random) # Generate a random password + # If $db_pwd is not provided, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" - # Create the user and grant access to the database - ynh_mongo_create_user --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name" + # Create the user and grant access to the database + ynh_mongo_create_user --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name" - # Store the password in the app's config - ynh_app_setting_set --key=db_pwd --value=$db_pwd + # Store the password in the app's config + ynh_app_setting_set --key=db_pwd --value=$db_pwd } # Remove a database if it exists, and the associated user # # usage: ynh_mongo_remove_db --db_user=user --db_name=name -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database # # ynh_mongo_remove_db() { # ============ Argument parsing ============= - local -A args_array=( [u]=db_user= [n]=db_name= ) - local db_user - local db_name - ynh_handle_getopts_args "$@" + local -A args_array=( [u]=db_user= [n]=db_name= ) + local db_user + local db_name + ynh_handle_getopts_args "$@" # =========================================== - if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists - ynh_mongo_drop_db --database=$db_name # Remove the database - else - ynh_print_warn --message="Database $db_name not found" - fi + if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists + ynh_mongo_drop_db --database=$db_name # Remove the database + else + ynh_print_warn --message="Database $db_name not found" + fi - # Remove mongo user if it exists - ynh_mongo_drop_user --db_user=$db_user --db_name=$db_name + # Remove mongo user if it exists + ynh_mongo_drop_user --db_user=$db_user --db_name=$db_name } # Install MongoDB and integrate MongoDB service in YunoHost # # usage: ynh_install_mongo [--mongo_version=mongo_version] -# | arg: -m, --mongo_version= - Version of MongoDB to install +# | arg: -m, --mongo_version= - Version of MongoDB to install # # ynh_install_mongo() { # ============ Argument parsing ============= - local -A args_array=([m]=mongo_version=) - local mongo_version - ynh_handle_getopts_args "$@" - mongo_version="${mongo_version:-$YNH_MONGO_VERSION}" + local -A args_array=([m]=mongo_version=) + local mongo_version + ynh_handle_getopts_args "$@" + mongo_version="${mongo_version:-$YNH_MONGO_VERSION}" # =========================================== - ynh_print_info --message="Installing MongoDB Community Edition ..." - local mongo_debian_release=$(ynh_get_debian_release) + ynh_print_info --message="Installing MongoDB Community Edition ..." + local mongo_debian_release=$(ynh_get_debian_release) - if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then + if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." mongo_version="4.4" fi @@ -309,19 +239,19 @@ ynh_install_mongo() { mongo_debian_release=buster fi - ynh_install_extra_app_dependencies --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc" - mongodb_servicename=mongod + ynh_install_extra_app_dependencies --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc" + mongodb_servicename=mongod - # Make sure MongoDB is started and enabled - systemctl enable $mongodb_servicename --quiet - systemctl daemon-reload --quiet - ynh_systemd_action --service_name=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" + # Make sure MongoDB is started and enabled + systemctl enable $mongodb_servicename --quiet + systemctl daemon-reload --quiet + ynh_systemd_action --service_name=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" - # Integrate MongoDB service in YunoHost - yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" + # Integrate MongoDB service in YunoHost + yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" - # Store mongo_version into the config of this app - ynh_app_setting_set --key=mongo_version --value=$mongo_version + # Store mongo_version into the config of this app + ynh_app_setting_set --key=mongo_version --value=$mongo_version } # Remove MongoDB @@ -332,14 +262,14 @@ ynh_install_mongo() { # # ynh_remove_mongo() { - # Only remove the mongodb service if it is not installed. - if ! ynh_package_is_installed "mongodb*" - then - ynh_print_info --message="Removing MongoDB service..." - mongodb_servicename=mongod - # Remove the mongodb service - yunohost service remove $mongodb_servicename - ynh_safe_rm --target="/var/lib/mongodb" - ynh_safe_rm --target="/var/log/mongodb" - fi + # Only remove the mongodb service if it is not installed. + if ! ynh_package_is_installed "mongodb*" + then + ynh_print_info --message="Removing MongoDB service..." + mongodb_servicename=mongod + # Remove the mongodb service + yunohost service remove $mongodb_servicename + ynh_safe_rm --target="/var/lib/mongodb" + ynh_safe_rm --target="/var/log/mongodb" + fi } From ef7da9e70fd658a3d41a901b2ed4dfac63126d9d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 14:51:40 +0200 Subject: [PATCH 034/146] Remove legacy references to path_url (instead of path) from packaging v1 era --- helpers/helpers.v2.1.d/nginx | 6 +++--- helpers/helpers.v2.1.d/templating | 15 ++++++++------- helpers/helpers.v2.1.d/utils | 8 ++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index b9f48dafe..b7ce7699a 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -9,8 +9,8 @@ # format and how placeholders are replaced with actual variables. # # Additionally, ynh_add_nginx_config will replace: -# - `#sub_path_only` by empty string if `path_url` is not `'/'` -# - `#root_path_only` by empty string if `path_url` *is* `'/'` +# - `#sub_path_only` by empty string if `path` is not `'/'` +# - `#root_path_only` by empty string if `path` *is* `'/'` # # This allows to enable/disable specific behaviors dependenging on the install # location @@ -22,7 +22,7 @@ ynh_add_nginx_config() { ynh_add_config --template="nginx.conf" --destination="$finalnginxconf" - if [ "${path_url:-}" != "/" ]; then + if [ "${path:-}" != "/" ]; then ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf" else ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf" diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index b70b73e97..3a83155f3 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -22,13 +22,13 @@ # `destination` by replacing the following keywords with global variables # that should be defined before calling this helper : # ``` -# __PATH__ by $path_url # __USER__ by $app # __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH # ``` # And any dynamic variables that should be defined before calling this helper like: # ``` # __DOMAIN__ by $domain +# __PATH__ by $path # __APP__ by $app # __VAR_1__ by $var_1 # __VAR_2__ by $var_2 @@ -125,7 +125,8 @@ ynh_add_config() { # # The helper will replace the following keywords with global variables # that should be defined before calling this helper : -# __PATH__ by $path_url +# __PATH__ by $path +# __PATH__/ by $path/ if $path != /, or just / otherwise (instead of //) # __USER__ by $app # __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH # @@ -144,11 +145,11 @@ ynh_replace_vars() { # =========================================== # Replace specific YunoHost variables - if test -n "${path_url:-}"; then - # path_url_slash_less is path_url, or a blank value if path_url is only '/' - local path_url_slash_less=${path_url%/} - ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file" - ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" + if test -n "${path:-}"; then + # path_slash_less is path, or a blank value if path is only '/' + local path_slash_less=${path%/} + ynh_replace_string --match_string="__PATH__/" --replace_string="$path_slash_less/" --target_file="$file" + ynh_replace_string --match_string="__PATH__" --replace_string="$path" --target_file="$file" fi if test -n "${app:-}"; then ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 5974c7d31..9e122cb6b 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -353,7 +353,7 @@ ynh_setup_source() { # Curl abstraction to help with POST requests to local pages (such as installation forms) # # usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ... -# | arg: page_uri - Path (relative to `$path_url`) of the page where POST data will be sent +# | arg: page_uri - Path (relative to `$path`) of the page where POST data will be sent # | arg: key1=value1 - (Optionnal) POST key and corresponding value # | arg: key2=value2 - (Optionnal) Another POST key and corresponding value # | arg: ... - (Optionnal) More POST keys and values @@ -362,15 +362,15 @@ ynh_setup_source() { # # For multiple calls, cookies are persisted between each call for the same app # -# `$domain` and `$path_url` should be defined externally (and correspond to the domain.tld and the /path (of the app?)) +# `$domain` and `$path` should be defined externally (and correspond to the domain.tld and the /path (of the app?)) # # Requires YunoHost version 2.6.4 or higher. ynh_local_curl() { # Define url of page to curl local local_page=$(ynh_normalize_url_path $1) - local full_path=$path_url$local_page + local full_path=$path$local_page - if [ "${path_url}" == "/" ]; then + if [ "${path}" == "/" ]; then full_path=$local_page fi From 6e2b36d9571450e35c6736b174266279d9f5b512 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 15:00:05 +0200 Subject: [PATCH 035/146] helpers2.1: remove unused options --label/--show_tile/--protected in ynh_permission_update --- helpers/helpers.v2.1.d/permission | 35 +++---------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/helpers/helpers.v2.1.d/permission b/helpers/helpers.v2.1.d/permission index adf9544ee..ab74f1f1b 100644 --- a/helpers/helpers.v2.1.d/permission +++ b/helpers/helpers.v2.1.d/permission @@ -258,30 +258,21 @@ ynh_permission_url() { # [packagingv1] # # usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]] -# [--label="label"] [--show_tile=true|false] [--protected=true|false] +# # | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) # | arg: -a, --add= - the list of group or users to enable add to the permission # | arg: -r, --remove= - the list of group or users to remove from the permission -# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. -# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO -# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. # # Requires YunoHost version 3.7.0 or higher. ynh_permission_update() { # ============ Argument parsing ============= - local -A args_array=([p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected=) + local -A args_array=([p]=permission= [a]=add= [r]=remove=) local permission local add local remove - local label - local show_tile - local protected ynh_handle_getopts_args "$@" add=${add:-} remove=${remove:-} - label=${label:-} - show_tile=${show_tile:-} - protected=${protected:-} # =========================================== if [[ -n $add ]]; then @@ -303,27 +294,7 @@ ynh_permission_update() { remove=",remove=['${remove//';'/"','"}']" fi - if [[ -n $label ]]; then - label=",label='$label'" - fi - - if [[ -n $show_tile ]]; then - if [ $show_tile == "true" ]; then - show_tile=",show_tile=True" - else - show_tile=",show_tile=False" - fi - fi - - if [[ -n $protected ]]; then - if [ $protected == "true" ]; then - protected=",protected=True" - else - protected=",protected=False" - fi - fi - - yunohost tools shell -c "from yunohost.permission import user_permission_update; user_permission_update('$app.$permission' $add $remove $label $show_tile $protected , force=True)" + yunohost tools shell -c "from yunohost.permission import user_permission_update; user_permission_update('$app.$permission' $add $remove , force=True)" } # Check if a permission has an user From cbc68afea406bb9d97199f46428e5d94a39470bc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 15:03:16 +0200 Subject: [PATCH 036/146] helpers2.1: remove old references to packaging v1 $final_path --- helpers/helpers.v2.1.d/config | 12 ++++++------ helpers/helpers.v2.1.d/nodejs | 2 +- helpers/helpers.v2.1.d/php | 4 ++-- helpers/helpers.v2.1.d/ruby | 4 ++-- helpers/helpers.v2.1.d/utils | 3 --- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index 4d4b74465..9e0127d36 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -22,7 +22,7 @@ _ynh_app_config_get_one() { if [[ "$bind" == "settings" ]]; then ynh_die --message="File '${short_setting}' can't be stored in settings" fi - old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" + old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" # Get multiline text from settings or from a full file @@ -32,7 +32,7 @@ _ynh_app_config_get_one() { elif [[ "$bind" == *":"* ]]; then ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else - old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" + old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" fi # Get value from a kind of key/value file @@ -47,7 +47,7 @@ _ynh_app_config_get_one() { bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" fi - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key_}" --after="${bind_after}")" fi @@ -73,7 +73,7 @@ _ynh_app_config_apply_one() { if [[ "$bind" == "settings" ]]; then ynh_die --message="File '${short_setting}' can't be stored in settings" fi - local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]]; then ynh_backup_if_checksum_is_different --file="$bind_file" ynh_safe_rm --target="$bind_file" @@ -98,7 +98,7 @@ _ynh_app_config_apply_one() { if [[ "$bind" == *":"* ]]; then ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi - local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" echo "${!short_setting}" >"$bind_file" ynh_store_file_checksum --file="$bind_file" --update_only @@ -113,7 +113,7 @@ _ynh_app_config_apply_one() { bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" fi bind_key_=${bind_key_:-$short_setting} - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s@__FINALPATH__@${final_path:-}@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}" diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index 72afd6b7a..d2ae2c1cc 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -25,7 +25,7 @@ export N_PREFIX="$n_install_dir" # However, $PATH is duplicated into $node_PATH to outlast any manipulation of `$PATH` # You can use the variable `$ynh_node_load_PATH` to quickly load your node version # in $PATH for an usage into a separate script. -# Exemple: $ynh_node_load_PATH $final_path/script_that_use_npm.sh` +# Exemple: $ynh_node_load_PATH $install_dir/script_that_use_npm.sh` # # # Finally, to start a nodejs service with the correct version, 2 solutions diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index a48b93ea5..158302962 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -289,7 +289,7 @@ YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} # Execute a command with Composer # # usage: ynh_composer_exec [--workdir=$install_dir] --commands="commands" -# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir # | arg: -c, --commands - Commands to execute. # # Requires YunoHost version 4.2 or higher. @@ -299,7 +299,7 @@ ynh_composer_exec() { local workdir local commands ynh_handle_getopts_args "$@" - workdir="${workdir:-${install_dir:-$final_path}}" + workdir="${workdir:-${install_dir}}" # =========================================== COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 1b4f193bb..1a03d1d2e 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -29,7 +29,7 @@ fi # However, $PATH is duplicated into $ruby_path to outlast any manipulation of $PATH # You can use the variable `$ynh_ruby_load_path` to quickly load your Ruby version # in $PATH for an usage into a separate script. -# Exemple: $ynh_ruby_load_path $final_path/script_that_use_gem.sh` +# Exemple: $ynh_ruby_load_path $install_dir/script_that_use_gem.sh` # # # Finally, to start a Ruby service with the correct version, 2 solutions @@ -74,7 +74,7 @@ ynh_use_ruby () { ynh_ruby_load_path="PATH=$PATH" # Sets the local application-specific Ruby version - pushd ${install_dir:-$final_path} + pushd ${install_dir} $rbenv_install_dir/bin/rbenv local $ruby_version popd } diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 9e122cb6b..cfe675144 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -268,9 +268,6 @@ ynh_setup_source() { if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then _ynh_apply_default_permissions $dest_dir fi - if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then - _ynh_apply_default_permissions $dest_dir - fi if [[ "$src_extract" == "false" ]]; then if [[ -z "$src_rename" ]] From f2f8b3e31956a2c7f16e3f10cd44faf40a8f251d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 16:30:28 +0200 Subject: [PATCH 037/146] helpers2.1: rework ynh_install_composer to wget the actual composer.phar instead of calling the unecessarily complex composer install script --- helpers/helpers.v2.1.d/php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 158302962..378ad2286 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -320,15 +320,21 @@ ynh_install_composer() { local workdir local install_args ynh_handle_getopts_args "$@" - # =========================================== - workdir="${workdir:-$install_dir}" install_args="${install_args:-}" + # =========================================== - curl -sS https://getcomposer.org/installer \ - | COMPOSER_HOME="$workdir/.composer" \ - php${phpversion} -- --quiet --install-dir="$workdir" --version=$YNH_COMPOSER_VERSION \ - || ynh_die --message="Unable to install Composer." + local composer_url="https://getcomposer.org/download/$YNH_COMPOSER_VERSION/composer.phar" + + [[ ! -e "$workdir/composer.phar" ]] || ynh_safe_rm $workdir/composer.phar + + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir $composer_url 2>&1) \ + || ynh_die --message="$out" # install dependencies ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \ From 576e35321f47e5a178ac8090784c833d3bce43ad Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 16:35:15 +0200 Subject: [PATCH 038/146] helpers2.1: replace $YNH_COMPOSER_VERSION with $composer_version to be consistent with other technologies: node_version, ruby_version, go_version etc... --- helpers/helpers.v2.1.d/php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 378ad2286..08850b2cc 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -281,11 +281,6 @@ ynh_get_scalable_phpfpm() { fi } -readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17 -# Declare the actual composer version to use. -# A packager willing to use another version of composer can override the variable into its _common.sh. -YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} - # Execute a command with Composer # # usage: ynh_composer_exec [--workdir=$install_dir] --commands="commands" @@ -324,10 +319,12 @@ ynh_install_composer() { install_args="${install_args:-}" # =========================================== - local composer_url="https://getcomposer.org/download/$YNH_COMPOSER_VERSION/composer.phar" + [[ -n "${composer_version}" ]] || ynh_die --message="\$composer_version should be defined before calling ynh_install_composer. (In the past, this was called \$YNH_COMPOSER_VERSION)" [[ ! -e "$workdir/composer.phar" ]] || ynh_safe_rm $workdir/composer.phar - + + local composer_url="https://getcomposer.org/download/$composer_version/composer.phar" + # NB. we have to declare the var as local first, # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work # because local always return 0 ... @@ -337,6 +334,6 @@ ynh_install_composer() { || ynh_die --message="$out" # install dependencies - ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \ + ynh_composer_exec --workdir="$workdir" --commands="install --no-dev $install_args" \ || ynh_die --message="Unable to install core dependencies with Composer." } From 05a02221b998ff5e5b74466829479100f0a05434 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 16:52:55 +0200 Subject: [PATCH 039/146] helpers2.1: for mysql and psql helpers, use db_name, db_pwd, db_name as default values --- helpers/helpers.v2.1.d/mysql | 21 ++++++++++++--------- helpers/helpers.v2.1.d/postgresql | 26 ++++++++++++++------------ 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index 1f91735ed..1196e5e07 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -3,9 +3,9 @@ # Open a connection as a user # # usage: ynh_mysql_connect_as --user=user --password=password [--database=database] -# | arg: -u, --user= - the user name to connect as -# | arg: -p, --password= - the user password -# | arg: -d, --database= - the database to connect to +# | arg: -u, --user= - the user name to connect as (by default, $db_user) +# | arg: -p, --password= - the user password (by default, $db_pwd) +# | arg: -d, --database= - the database to connect to (by default, $db_name) # # examples: # ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;" @@ -19,7 +19,9 @@ ynh_mysql_connect_as() { local password local database ynh_handle_getopts_args "$@" - database="${database:-}" + user="${database:-$db_name}" + password="${database:-$db_pwd}" + database="${database:-$db_name}" # =========================================== mysql --user="$user" --password="$password" --batch "$database" @@ -29,7 +31,7 @@ ynh_mysql_connect_as() { # # usage: ynh_mysql_execute_as_root --sql=sql [--database=database] # | arg: -s, --sql= - the SQL command to execute -# | arg: -d, --database= - the database to connect to +# | arg: -d, --database= - the database to connect to (by default, $db_name) # # Requires YunoHost version 2.2.4 or higher. ynh_mysql_execute_as_root() { @@ -38,7 +40,7 @@ ynh_mysql_execute_as_root() { local sql local database ynh_handle_getopts_args "$@" - database="${database:-}" + database="${database:-$db_name}" # =========================================== if [ -n "$database" ]; then @@ -52,7 +54,7 @@ ynh_mysql_execute_as_root() { # # usage: ynh_mysql_execute_file_as_root --file=file [--database=database] # | arg: -f, --file= - the file containing SQL commands -# | arg: -d, --database= - the database to connect to +# | arg: -d, --database= - the database to connect to (by default, $db_name) # # Requires YunoHost version 2.2.4 or higher. ynh_mysql_execute_file_as_root() { @@ -61,7 +63,7 @@ ynh_mysql_execute_file_as_root() { local file local database ynh_handle_getopts_args "$@" - database="${database:-}" + database="${database:-$db_name}" # =========================================== if [ -n "$database" ]; then @@ -116,7 +118,7 @@ ynh_mysql_drop_db() { # Dump a database # # usage: ynh_mysql_dump_db --database=database -# | arg: -d, --database= - the database name to dump +# | arg: -d, --database= - the database name to dump (by default, $db_name) # | ret: The mysqldump output # # example: ynh_mysql_dump_db --database=roundcube > ./dump.sql @@ -127,6 +129,7 @@ ynh_mysql_dump_db() { local -A args_array=([d]=database=) local database ynh_handle_getopts_args "$@" + database="${database:-$db_name}" # =========================================== mysqldump --single-transaction --skip-dump-date --routines "$database" diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 71be36f8f..9b27697c4 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -6,9 +6,9 @@ PSQL_VERSION=13 # Open a connection as a user # # usage: ynh_psql_connect_as --user=user --password=password [--database=database] -# | arg: -u, --user= - the user name to connect as -# | arg: -p, --password= - the user password -# | arg: -d, --database= - the database to connect to +# | arg: -u, --user= - the user name to connect as (by default, $db_user) +# | arg: -p, --password= - the user password (by default, $db_pw) +# | arg: -d, --database= - the database to connect to (by default, $db_name) # # examples: # ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" @@ -22,7 +22,9 @@ ynh_psql_connect_as() { local password local database ynh_handle_getopts_args "$@" - database="${database:-}" + user="${user:-$db_user}" + password="${password:-$db_pwd}" + database="${database:-$db_name}" # =========================================== sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" @@ -32,7 +34,7 @@ ynh_psql_connect_as() { # # usage: ynh_psql_execute_as_root --sql=sql [--database=database] # | arg: -s, --sql= - the SQL command to execute -# | arg: -d, --database= - the database to connect to +# | arg: -d, --database= - the database to connect to (by default, $db_name) # # Requires YunoHost version 3.5.0 or higher. ynh_psql_execute_as_root() { @@ -41,7 +43,7 @@ ynh_psql_execute_as_root() { local sql local database ynh_handle_getopts_args "$@" - database="${database:-}" + database="${database:-$db_name}" # =========================================== if [ -n "$database" ]; then @@ -56,7 +58,7 @@ ynh_psql_execute_as_root() { # # usage: ynh_psql_execute_file_as_root --file=file [--database=database] # | arg: -f, --file= - the file containing SQL commands -# | arg: -d, --database= - the database to connect to +# | arg: -d, --database= - the database to connect to (by default, $db_name) # # Requires YunoHost version 3.5.0 or higher. ynh_psql_execute_file_as_root() { @@ -65,7 +67,7 @@ ynh_psql_execute_file_as_root() { local file local database ynh_handle_getopts_args "$@" - database="${database:-}" + database="${database:-$db_name}" # =========================================== if [ -n "$database" ]; then @@ -123,7 +125,7 @@ ynh_psql_drop_db() { # Dump a database # # usage: ynh_psql_dump_db --database=database -# | arg: -d, --database= - the database name to dump +# | arg: -d, --database= - the database name to dump (by default, $db_name) # | ret: the psqldump output # # example: ynh_psql_dump_db 'roundcube' > ./dump.sql @@ -134,6 +136,7 @@ ynh_psql_dump_db() { local -A args_array=([d]=database=) local database ynh_handle_getopts_args "$@" + database="${database:-$db_name}" # =========================================== sudo --login --user=postgres pg_dump "$database" @@ -180,7 +183,7 @@ ynh_psql_user_exists() { # Check if a psql database exists # # usage: ynh_psql_database_exists --database=database -# | arg: -d, --database= - the database for which to check existence +# | arg: -d, --database= - the database for which to check existence (by default, $db_name) # | exit: Return 1 if the database doesn't exist, 0 otherwise # # Requires YunoHost version 3.5.0 or higher. @@ -189,6 +192,7 @@ ynh_psql_database_exists() { local -A args_array=([d]=database=) local database ynh_handle_getopts_args "$@" + database="${database:-$db_name}" # =========================================== # if psql is not there, we cannot check the db @@ -226,7 +230,6 @@ ynh_psql_drop_user() { # | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated # # After executing this helper, the password of the created database will be available in $db_pwd -# It will also be stored as "psqlpwd" into the app settings. # # Requires YunoHost version 2.7.13 or higher. ynh_psql_setup_db() { @@ -249,7 +252,6 @@ ynh_psql_setup_db() { fi ynh_psql_create_db "$db_name" "$db_user" # Create the database - ynh_app_setting_set --key=psqlpwd --value=$db_pwd # Store the password in the app's config } # Remove a database if it exists, and the associated user From 4d5ae9d32c4fa98269ba9b21efba74818b04f877 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 17:00:21 +0200 Subject: [PATCH 040/146] helpers2.1: mysql/psql: rename ynh_SQL_connect_as to ynh_SQL_execute for semantics and consistency with ynh_SQL_execute_as_root --- helpers/helpers.v2.1.d/mysql | 8 ++++---- helpers/helpers.v2.1.d/postgresql | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index 1196e5e07..e8cf9a69d 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -2,17 +2,17 @@ # Open a connection as a user # -# usage: ynh_mysql_connect_as --user=user --password=password [--database=database] +# usage: ynh_mysql_execute --user=user --password=password [--database=database] # | arg: -u, --user= - the user name to connect as (by default, $db_user) # | arg: -p, --password= - the user password (by default, $db_pwd) # | arg: -d, --database= - the database to connect to (by default, $db_name) # # examples: -# ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;" -# ynh_mysql_connect_as --user="user" --password="pass" < /path/to/file.sql +# ynh_mysql_execute <<< "UPDATE ...;" +# ynh_mysql_execute < /path/to/file.sql # # Requires YunoHost version 2.2.4 or higher. -ynh_mysql_connect_as() { +ynh_mysql_execute() { # ============ Argument parsing ============= local -A args_array=([u]=user= [p]=password= [d]=database=) local user diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 9b27697c4..4e2a3bb24 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -5,17 +5,17 @@ PSQL_VERSION=13 # Open a connection as a user # -# usage: ynh_psql_connect_as --user=user --password=password [--database=database] +# usage: ynh_psql_execute --user=user --password=password [--database=database] # | arg: -u, --user= - the user name to connect as (by default, $db_user) # | arg: -p, --password= - the user password (by default, $db_pw) # | arg: -d, --database= - the database to connect to (by default, $db_name) # # examples: -# ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" -# ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql +# ynh_psql_execute <<< "UPDATE ...;" +# ynh_psql_execute < /path/to/file.sql # # Requires YunoHost version 3.5.0 or higher. -ynh_psql_connect_as() { +ynh_psql_execute() { # ============ Argument parsing ============= local -A args_array=([u]=user= [p]=password= [d]=database=) local user @@ -50,8 +50,8 @@ ynh_psql_execute_as_root() { database="--database=$database" fi - ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ - $database <<<"$sql" + ynh_psql_execute --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" <<<"$sql" } # Execute a command from a file as root user @@ -74,8 +74,8 @@ ynh_psql_execute_file_as_root() { database="--database=$database" fi - ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ - $database <"$file" + ynh_psql_execute --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" <"$file" } # Create a database and grant optionnaly privilegies to a user From 51d1011c476a3d5d452b915e52fb1f053f103bbd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:00:20 +0200 Subject: [PATCH 041/146] helpers2.1: Remove the --foo_version arg for go, mongodb, nodejs, php, ruby helpers. Just use the global $foo_version which should be defined as global --- helpers/helpers.v2.1.d/go | 18 ++++++++---------- helpers/helpers.v2.1.d/mongodb | 26 +++++++++++--------------- helpers/helpers.v2.1.d/nodejs | 20 ++++++++------------ helpers/helpers.v2.1.d/php | 2 ++ helpers/helpers.v2.1.d/ruby | 20 +++++++++----------- 5 files changed, 38 insertions(+), 48 deletions(-) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index d6b74aaf9..98def7f3f 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -52,7 +52,8 @@ export GOENV_ROOT="$goenv_install_dir" # # Requires YunoHost version 3.2.2 or higher. ynh_use_go () { - go_version=$(ynh_app_setting_get --key=go_version) + + [[ -n "${go_version:-}" ]] || ynh_die --message="\$go_version should be defined prior to calling ynh_use_go" # Get the absolute path of this version of Go go_path="$go_version_path/$go_version/bin" @@ -77,9 +78,9 @@ ynh_use_go () { popd } -# Install a specific version of Go +# Install a specific version of Go using goenv # -# ynh_install_go will install the version of Go provided as argument by using goenv. +# The installed version is defined by $nodejs_version which should be defined as global prior to calling this helper # # This helper creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv # for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) @@ -89,16 +90,12 @@ ynh_use_go () { # When not possible (e.g. in systemd service definition), please use direct path # to goenv shims (e.g. $goenv_ROOT/shims/bundle) # -# usage: ynh_install_go --go_version=go_version -# | arg: -v, --go_version= - Version of go to install. +# usage: ynh_install_go # # Requires YunoHost version 3.2.2 or higher. ynh_install_go () { - # ============ Argument parsing ============= - local -A args_array=( [v]=go_version= ) - local go_version - ynh_handle_getopts_args "$@" - # =========================================== + + [[ -n "${go_version:-}" ]] || ynh_die --message="\$go_version should be defined prior to calling ynh_install_go" # Load goenv path in PATH local CLEAR_PATH="$goenv_install_dir/bin:$PATH" @@ -160,6 +157,7 @@ ynh_install_go () { # Store go_version into the config of this app ynh_app_setting_set --app="$app" --key="go_version" --value="$final_go_version" + go_version=$final_go_version # Cleanup Go versions ynh_cleanup_go diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 1626c27d2..226c21084 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -215,29 +215,25 @@ ynh_mongo_remove_db() { # Install MongoDB and integrate MongoDB service in YunoHost # -# usage: ynh_install_mongo [--mongo_version=mongo_version] -# | arg: -m, --mongo_version= - Version of MongoDB to install +# The installed version is defined by $mongo_version which should be defined as global prior to calling this helper # +# usage: ynh_install_mongo # ynh_install_mongo() { - # ============ Argument parsing ============= - local -A args_array=([m]=mongo_version=) - local mongo_version - ynh_handle_getopts_args "$@" - mongo_version="${mongo_version:-$YNH_MONGO_VERSION}" - # =========================================== + + [[ -n "${mongo_version:-}" ]] || ynh_die --message="\$mongo_version should be defined prior to calling ynh_install_mongo" ynh_print_info --message="Installing MongoDB Community Edition ..." local mongo_debian_release=$(ynh_get_debian_release) if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then - ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." - mongo_version="4.4" - fi - if [[ "$mongo_version" == "4.4" ]]; then - ynh_print_warn --message="Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release." - mongo_debian_release=buster - fi + ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." + mongo_version="4.4" + fi + if [[ "$mongo_version" == "4.4" ]]; then + ynh_print_warn --message="Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release." + mongo_debian_release=buster + fi ynh_install_extra_app_dependencies --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc" mongodb_servicename=mongod diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index d2ae2c1cc..a151a54d8 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -46,11 +46,11 @@ export N_PREFIX="$n_install_dir" # # 2 other variables are also available # - $nodejs_path: The absolute path to node binaries for the chosen version. -# - $nodejs_version: Just the version number of node for this app. Stored as 'nodejs_version' in settings.yml. # # Requires YunoHost version 2.7.12 or higher. ynh_use_nodejs() { - nodejs_version=$(ynh_app_setting_get --key=nodejs_version) + + [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_install_nodejs" # Get the absolute path of this version of node nodejs_path="$node_version_path/$nodejs_version/bin" @@ -78,12 +78,11 @@ ynh_use_nodejs() { export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 } -# Install a specific version of nodejs +# Install a specific version of nodejs, using 'n' # -# ynh_install_nodejs will install the version of node provided as argument by using n. +# The installed version is defined by $nodejs_version which should be defined as global prior to calling this helper # -# usage: ynh_install_nodejs --nodejs_version=nodejs_version -# | arg: -n, --nodejs_version= - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0). The crontab will then handle the update of minor versions when needed. +# usage: ynh_install_nodejs # # `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use. # That's how it changes the version @@ -94,11 +93,7 @@ ynh_use_nodejs() { ynh_install_nodejs() { # Use n, https://github.com/tj/n to manage the nodejs versions - # ============ Argument parsing ============= - local -A args_array=([n]=nodejs_version=) - local nodejs_version - ynh_handle_getopts_args "$@" - # =========================================== + [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_install_nodejs" # Create $n_install_dir mkdir --parents "$n_install_dir" @@ -164,7 +159,8 @@ ynh_install_nodejs() { # # Requires YunoHost version 2.7.12 or higher. ynh_remove_nodejs() { - nodejs_version=$(ynh_app_setting_get --key=nodejs_version) + + [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_install_nodejs" # Remove the line for this app sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version" diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 08850b2cc..e09a2dc4f 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -304,6 +304,8 @@ ynh_composer_exec() { # Install and initialize Composer in the given directory # +# The installed version is defined by $composer_version which should be defined as global prior to calling this helper +# # usage: ynh_install_composer [--workdir=$install_dir] [--install_args="--optimize-autoloader"] # | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. # | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 1a03d1d2e..f2092f070 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -51,7 +51,8 @@ fi # # Requires YunoHost version 3.2.2 or higher. ynh_use_ruby () { - ruby_version=$(ynh_app_setting_get --key=ruby_version) + + [[ -n "${ruby_version:-}" ]] || ynh_die --message="\$ruby_version should be defined prior to calling ynh_use_ruby" # Get the absolute path of this version of Ruby ruby_path="$ruby_version_path/$YNH_APP_INSTANCE_NAME/bin" @@ -79,9 +80,9 @@ ynh_use_ruby () { popd } -# Install a specific version of Ruby +# Install a specific version of Ruby using rbenv # -# ynh_install_ruby will install the version of Ruby provided as argument by using rbenv. +# The installed version is defined by $ruby_version which should be defined as global prior to calling this helper # # This helper creates a /etc/profile.d/rbenv.sh that configures PATH environment for rbenv # for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) @@ -91,16 +92,12 @@ ynh_use_ruby () { # When not possible (e.g. in systemd service definition), please use direct path # to rbenv shims (e.g. $RBENV_ROOT/shims/bundle) # -# usage: ynh_install_ruby --ruby_version=ruby_version -# | arg: -v, --ruby_version= - Version of ruby to install. +# usage: ynh_install_ruby # # Requires YunoHost version 3.2.2 or higher. ynh_install_ruby () { - # ============ Argument parsing ============= - local -A args_array=( [v]=ruby_version= ) - local ruby_version - ynh_handle_getopts_args "$@" - # =========================================== + + [[ -n "${ruby_version:-}" ]] || ynh_die --message="\$ruby_version should be defined prior to calling ynh_install_ruby" # Load rbenv path in PATH local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" @@ -207,6 +204,7 @@ ynh_install_ruby () { # Store ruby_version into the config of this app ynh_app_setting_set --key=ruby_version --value=$final_ruby_version + ruby_version=$final_ruby_version # Remove app virtualenv if rbenv alias --list | grep --quiet "$YNH_APP_INSTANCE_NAME " @@ -274,7 +272,7 @@ ynh_cleanup_ruby () { required_ruby_versions="${installed_app_ruby_version}\n${required_ruby_versions}" fi done - + # Remove no more needed Ruby versions local installed_ruby_versions=$(rbenv versions --bare --skip-aliases | grep -Ev '/') for installed_ruby_version in $installed_ruby_versions From 29d6dd685afa90e8c51fbf84ee4caf1ac9b22b06 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:01:17 +0200 Subject: [PATCH 042/146] helpers2.1: Remove weird old packaging v1 trick for apt dependencies related to ruby --- helpers/helpers.v2.1.d/ruby | 5 ----- 1 file changed, 5 deletions(-) diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index f2092f070..8e19548c7 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -7,11 +7,6 @@ ruby_version_path="$rbenv_install_dir/versions" export RBENV_ROOT="$rbenv_install_dir" export rbenv_root="$rbenv_install_dir" -if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - build_ruby_dependencies="libjemalloc-dev curl build-essential libreadline-dev zlib1g-dev libsqlite3-dev libssl-dev libxml2-dev libxslt-dev autoconf automake bison libtool" - build_pkg_dependencies="${build_pkg_dependencies:-} $build_ruby_dependencies" -fi - # Load the version of Ruby for an app, and set variables. # # ynh_use_ruby has to be used in any app scripts before using Ruby for the first time. From 14ba98a232adfb091ac4a64f36ce8abd7d076a84 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:05:23 +0200 Subject: [PATCH 043/146] helpers2.1: remove legacy code in setting handling --- helpers/helpers.v2.1.d/setting | 2 -- 1 file changed, 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 60dbf0efc..0d737491e 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -85,8 +85,6 @@ else: if key in settings: del settings[key] elif action == "set": - if key in ['redirected_urls', 'redirected_regex']: - value = yaml.safe_load(value) settings[key] = value else: raise ValueError("action should either be get, set or delete") From 1b2d13f96a16f49ab4d6417a57eed23f6cd7a365 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:18:42 +0200 Subject: [PATCH 044/146] helpers2.1: simplify ynh_replace_string, just write ynh_replace --match=foo --replace=bar --file=/some/path --- helpers/helpers.v2.1.d/nginx | 4 +-- helpers/helpers.v2.1.d/nodejs | 2 +- helpers/helpers.v2.1.d/string | 52 +++++++++++++++---------------- helpers/helpers.v2.1.d/templating | 8 ++--- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index b7ce7699a..c6a6cfce1 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -23,9 +23,9 @@ ynh_add_nginx_config() { ynh_add_config --template="nginx.conf" --destination="$finalnginxconf" if [ "${path:-}" != "/" ]; then - ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf" + ynh_replace --match="^#sub_path_only" --replace="" --file="$finalnginxconf" else - ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf" + ynh_replace --match="^#root_path_only" --replace="" --file="$finalnginxconf" fi ynh_store_file_checksum --file="$finalnginxconf" diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index a151a54d8..c8dfb94c7 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -111,7 +111,7 @@ ynh_install_nodejs() { mkdir -p $n_install_dir/bin/ cp "$YNH_HELPERS_DIR/vendor/n/n" $n_install_dir/bin/n # Tweak for n to understand it's installed in $N_PREFIX - ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n" + ynh_replace --match="^N_PREFIX=\${N_PREFIX-.*}$" --replace="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --file="$n_install_dir/bin/n" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH diff --git a/helpers/helpers.v2.1.d/string b/helpers/helpers.v2.1.d/string index 7990e2b19..bacb95f93 100644 --- a/helpers/helpers.v2.1.d/string +++ b/helpers/helpers.v2.1.d/string @@ -27,63 +27,63 @@ ynh_string_random() { # Substitute/replace a string (or expression) by another in a file # -# usage: ynh_replace_string --match_string=match_string --replace_string=replace_string --target_file=target_file -# | arg: -m, --match_string= - String to be searched and replaced in the file -# | arg: -r, --replace_string= - String that will replace matches -# | arg: -f, --target_file= - File in which the string will be replaced. +# usage: ynh_replace --match=match --replace=replace --file=file +# | arg: -m, --match= - String to be searched and replaced in the file +# | arg: -r, --replace= - String that will replace matches +# | arg: -f, --file= - File in which the string will be replaced. # # As this helper is based on sed command, regular expressions and references to # sub-expressions can be used (see sed manual page for more information) # # Requires YunoHost version 2.6.4 or higher. -ynh_replace_string() { +ynh_replace() { # ============ Argument parsing ============= - local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) - local match_string - local replace_string - local target_file + local -A args_array=([m]=match= [r]=replaceg= [f]=file=) + local match + local replace + local file ynh_handle_getopts_args "$@" # =========================================== set +o xtrace # set +x local delimit=$'\001' # Escape the delimiter if it's in the string. - match_string=${match_string//${delimit}/"\\${delimit}"} - replace_string=${replace_string//${delimit}/"\\${delimit}"} + match=${match//${delimit}/"\\${delimit}"} + replace=${replace//${delimit}/"\\${delimit}"} set -o xtrace # set -x - sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file" + sed --in-place "s${delimit}${match}${delimit}${replace}${delimit}g" "$file" } # Substitute/replace a special string by another in a file # -# usage: ynh_replace_special_string --match_string=match_string --replace_string=replace_string --target_file=target_file -# | arg: -m, --match_string= - String to be searched and replaced in the file -# | arg: -r, --replace_string= - String that will replace matches -# | arg: -t, --target_file= - File in which the string will be replaced. +# usage: ynh_replace_special_string --match=match --replace=replace --file=file +# | arg: -m, --match= - String to be searched and replaced in the file +# | arg: -r, --replace= - String that will replace matches +# | arg: -f, --file= - File in which the string will be replaced. # -# This helper will use ynh_replace_string, but as you can use special +# This helper will use ynh_replace, but as you can use special # characters, you can't use some regular expressions and sub-expressions. # # Requires YunoHost version 2.7.7 or higher. ynh_replace_special_string() { # ============ Argument parsing ============= - local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) - local match_string - local replace_string - local target_file + local -A args_array=([m]=match= [r]=replace= [f]=file=) + local match + local replace + local file ynh_handle_getopts_args "$@" # =========================================== # Escape any backslash to preserve them as simple backslash. - match_string=${match_string//\\/"\\\\"} - replace_string=${replace_string//\\/"\\\\"} + match=${match//\\/"\\\\"} + replace=${replace//\\/"\\\\"} # Escape the & character, who has a special function in sed. - match_string=${match_string//&/"\&"} - replace_string=${replace_string//&/"\&"} + match=${match//&/"\&"} + replace=${replace//&/"\&"} - ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file" + ynh_replace --match="$match" --replace="$replace" --file="$file" } # Sanitize a string intended to be the name of a database diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index 3a83155f3..4f5bf25d0 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -148,14 +148,14 @@ ynh_replace_vars() { if test -n "${path:-}"; then # path_slash_less is path, or a blank value if path is only '/' local path_slash_less=${path%/} - ynh_replace_string --match_string="__PATH__/" --replace_string="$path_slash_less/" --target_file="$file" - ynh_replace_string --match_string="__PATH__" --replace_string="$path" --target_file="$file" + ynh_replace --match="__PATH__/" --replace="$path_slash_less/" --file="$file" + ynh_replace --match="__PATH__" --replace="$path" --file="$file" fi if test -n "${app:-}"; then - ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" + ynh_replace --match="__USER__" --replace="$app" --file="$file" fi if test -n "${ynh_node_load_PATH:-}"; then - ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" + ynh_replace --match="__YNH_NODE_LOAD_PATH__" --replace="$ynh_node_load_PATH" --file="$file" fi # Replace others variables From 2a6a8af0f7bd75089fdb2e3223a6976e531fa8ea Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:26:19 +0200 Subject: [PATCH 045/146] helpers2.1: ynh_systemd_action: rename --service_name to --service to be consistent with other commands --- helpers/helpers.v2.1.d/fail2ban | 4 ++-- helpers/helpers.v2.1.d/mongodb | 2 +- helpers/helpers.v2.1.d/nginx | 4 ++-- helpers/helpers.v2.1.d/php | 4 ++-- helpers/helpers.v2.1.d/systemd | 34 ++++++++++++++++----------------- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index 05fe0ea2b..580db74c9 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -105,7 +105,7 @@ ignoreregex = chown -R "$app:$app" "/var/log/$app" chmod -R u=rwX,g=rX,o= "/var/log/$app" - ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + ynh_systemd_action --service=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then @@ -122,5 +122,5 @@ ignoreregex = ynh_remove_fail2ban_config() { ynh_safe_rm --target="/etc/fail2ban/jail.d/$app.conf" ynh_safe_rm --target="/etc/fail2ban/filter.d/$app.conf" - ynh_systemd_action --service_name=fail2ban --action=reload + ynh_systemd_action --service=fail2ban --action=reload } diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 226c21084..62acc81e4 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -241,7 +241,7 @@ ynh_install_mongo() { # Make sure MongoDB is started and enabled systemctl enable $mongodb_servicename --quiet systemctl daemon-reload --quiet - ynh_systemd_action --service_name=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" + ynh_systemd_action --service=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" # Integrate MongoDB service in YunoHost yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index c6a6cfce1..50b5a1a17 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -30,7 +30,7 @@ ynh_add_nginx_config() { ynh_store_file_checksum --file="$finalnginxconf" - ynh_systemd_action --service_name=nginx --action=reload + ynh_systemd_action --service=nginx --action=reload } # Remove the dedicated nginx config @@ -40,7 +40,7 @@ ynh_add_nginx_config() { # Requires YunoHost version 2.7.2 or higher. ynh_remove_nginx_config() { ynh_safe_rm --target="/etc/nginx/conf.d/$domain.d/$app.conf" - ynh_systemd_action --service_name=nginx --action=reload + ynh_systemd_action --service=nginx --action=reload } diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index e09a2dc4f..41097956e 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -130,7 +130,7 @@ pm.process_idle_timeout = 10s ynh_safe_rm --target="$finalphpconf" ynh_die --message="The new configuration broke php-fpm?" fi - ynh_systemd_action --service_name=$fpm_service --action=reload + ynh_systemd_action --service=$fpm_service --action=reload } # Remove the dedicated PHP-FPM config @@ -143,7 +143,7 @@ ynh_remove_fpm_config() { local fpm_service=$(ynh_app_setting_get --key=fpm_service) ynh_safe_rm --target="$fpm_config_dir/pool.d/$app.conf" - ynh_systemd_action --service_name=$fpm_service --action=reload + ynh_systemd_action --service=$fpm_service --action=reload } # Define the values to configure PHP-FPM diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 8d63006fe..a40d4afb8 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -44,7 +44,7 @@ ynh_remove_systemd_config() { local finalsystemdconf="/etc/systemd/system/$service.service" if [ -e "$finalsystemdconf" ]; then - ynh_systemd_action --service_name=$service --action=stop + ynh_systemd_action --service=$service --action=stop systemctl disable $service --quiet ynh_safe_rm --target="$finalsystemdconf" systemctl daemon-reload @@ -53,8 +53,8 @@ ynh_remove_systemd_config() { # Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started # -# usage: ynh_systemd_action [--service_name=service_name] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] -# | arg: -n, --service_name= - Name of the service to start. Default : `$app` +# usage: ynh_systemd_action [--service=service] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] +# | arg: -n, --service= - Name of the service to start. Default : `$app` # | arg: -a, --action= - Action to perform with systemctl. Default: start # | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. # | arg: -p, --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` @@ -64,24 +64,24 @@ ynh_remove_systemd_config() { # Requires YunoHost version 3.5.0 or higher. ynh_systemd_action() { # ============ Argument parsing ============= - local -A args_array=([n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=) - local service_name + local -A args_array=([n]=service= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=) + local service local action local line_match local length local log_path local timeout ynh_handle_getopts_args "$@" - service_name="${service_name:-$app}" + service="${service:-$app}" action=${action:-start} line_match=${line_match:-} length=${length:-20} - log_path="${log_path:-/var/log/$service_name/$service_name.log}" + log_path="${log_path:-/var/log/$service/$service.log}" timeout=${timeout:-300} # =========================================== # Manage case of service already stopped - if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service_name; then + if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service; then return 0 fi @@ -91,7 +91,7 @@ ynh_systemd_action() { # Following the starting of the app in its log if [ "$log_path" == "systemd" ]; then # Read the systemd journal - journalctl --unit=$service_name --follow --since=-0 --quiet >"$templog" & + journalctl --unit=$service --follow --since=-0 --quiet >"$templog" & # Get the PID of the journalctl command local pid_tail=$! else @@ -110,9 +110,9 @@ ynh_systemd_action() { local time_start="$(date --utc --rfc-3339=seconds | cut -d+ -f1) UTC" # If the service fails to perform the action - if ! systemctl $action $service_name; then + if ! systemctl $action $service; then # Show syslog for this service - journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name >&2 + journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service >&2 # If a log is specified for this service, show also the content of this log if [ -e "$log_path" ]; then tail --lines=$length "$log_path" >&2 @@ -130,13 +130,13 @@ ynh_systemd_action() { # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout if [ "$log_path" == "systemd" ]; then # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action - if journalctl --unit=$service_name --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then - ynh_print_info --message="The service $service_name has correctly executed the action ${action}." + if journalctl --unit=$service --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then + ynh_print_info --message="The service $service has correctly executed the action ${action}." break fi else if grep --extended-regexp --quiet "$line_match" "$templog"; then - ynh_print_info --message="The service $service_name has correctly executed the action ${action}." + ynh_print_info --message="The service $service has correctly executed the action ${action}." break fi fi @@ -158,9 +158,9 @@ ynh_systemd_action() { echo "" >&2 fi if [ $i -eq $timeout ]; then - ynh_print_warn --message="The service $service_name didn't fully executed the action ${action} before the timeout." - ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" - journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name >&2 + ynh_print_warn --message="The service $service didn't fully executed the action ${action} before the timeout." + ynh_print_warn --message="Please find here an extract of the end of the log of the service $service:" + journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service >&2 if [ -e "$log_path" ]; then ynh_print_warn --message="\-\-\-" tail --lines=$length "$log_path" >&2 From 8ad3a3bc6fcec8bcc0fc59385cae67e84df8b0bf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:31:58 +0200 Subject: [PATCH 046/146] helpers2.1: remove old internal ynh_render_template, should use ynh_add_config --jinja instead --- helpers/helpers.v2.1.d/templating | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index 4f5bf25d0..64f9beb53 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -386,23 +386,3 @@ ynh_write_var_in_file() { fi set -o xtrace # set -x } - -# Render templates with Jinja2 -# -# [internal] -# -# Attention : Variables should be exported before calling this helper to be -# accessible inside templates. -# -# usage: ynh_render_template some_template output_path -# | arg: some_template - Template file to be rendered -# | arg: output_path - The path where the output will be redirected to -ynh_render_template() { - local template_path=$1 - local output_path=$2 - mkdir -p "$(dirname $output_path)" - # Taken from https://stackoverflow.com/a/35009576 - python3 -c 'import os, sys, jinja2; sys.stdout.write( - jinja2.Template(sys.stdin.read() - ).render(os.environ));' <$template_path >$output_path -} From a123149bef728fe2e7b0ad0992f847effd588045 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:36:25 +0200 Subject: [PATCH 047/146] helpers2.1: remove boring edge case in ynh_exec_as checking if asking to run as root..? --- helpers/helpers.v2.1.d/user | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index 7ab2301fa..c971a415b 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -180,10 +180,5 @@ ynh_system_user_delete() { ynh_exec_as() { local user=$1 shift 1 - - if [[ $user = $(whoami) ]]; then - eval "$@" - else - sudo -u "$user" "$@" - fi + sudo -u "$user" "$@" } From 0ceb77ec348b0c9cd686712e396e1e9f4f1714a2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:39:06 +0200 Subject: [PATCH 048/146] helpers2.1: ynh_setup_source: --full_replace now to be a boolean, no need to write --full_replace=1 --- helpers/helpers.v2.1.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index cfe675144..e3064e165 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -138,7 +138,7 @@ fi # Requires YunoHost version 2.6.4 or higher. ynh_setup_source() { # ============ Argument parsing ============= - local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) + local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace) local dest_dir local source_id local keep From 8c3ca4a0f4687e42eedc410c6507d2f2533effc1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:43:37 +0200 Subject: [PATCH 049/146] helpers2.1: $YNH_APP_INSTANCE_NAME -> $app --- helpers/helpers.v2.1.d/ruby | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 8e19548c7..eb8f8dc37 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -50,7 +50,7 @@ ynh_use_ruby () { [[ -n "${ruby_version:-}" ]] || ynh_die --message="\$ruby_version should be defined prior to calling ynh_use_ruby" # Get the absolute path of this version of Ruby - ruby_path="$ruby_version_path/$YNH_APP_INSTANCE_NAME/bin" + ruby_path="$ruby_version_path/$app/bin" # Allow alias to be used into bash script shopt -s expand_aliases @@ -202,13 +202,13 @@ ynh_install_ruby () { ruby_version=$final_ruby_version # Remove app virtualenv - if rbenv alias --list | grep --quiet "$YNH_APP_INSTANCE_NAME " + if rbenv alias --list | grep --quiet "$app " then - rbenv alias $YNH_APP_INSTANCE_NAME --remove + rbenv alias $app --remove fi # Create app virtualenv - rbenv alias $YNH_APP_INSTANCE_NAME $final_ruby_version + rbenv alias $app $final_ruby_version # Cleanup Ruby versions ynh_cleanup_ruby @@ -238,7 +238,7 @@ ynh_remove_ruby () { # Remove /usr/local/bin in PATH in case of Ruby prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') - rbenv alias $YNH_APP_INSTANCE_NAME --remove + rbenv alias $app --remove # Remove the line for this app ynh_app_setting_delete --key=ruby_version From d1e1fd5e35e72d1aaf6868481db584e9c4a5ab4d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 18:52:29 +0200 Subject: [PATCH 050/146] helpers2.1: rename ynh_app_upgrading_from_version_prior_to X -> _before X --- helpers/helpers.v2.1.d/utils | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index e3064e165..3901f04b8 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -502,10 +502,10 @@ ynh_check_app_version_changed() { # Compare the current package version is strictly lower than another version given as an argument # -# example: if ynh_app_upgrading_from_version_prior_to 2.3.2~ynh1; then ... +# example: if ynh_app_upgrading_from_version_before 2.3.2~ynh1; then ... # # Requires YunoHost version 11.2 or higher. -ynh_app_upgrading_from_version_prior_to() { +ynh_app_upgrading_from_version_before() { local version=$1 [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" @@ -514,10 +514,10 @@ ynh_app_upgrading_from_version_prior_to() { # Compare the current package version is lower or equal to another version given as an argument # -# example: if ynh_app_upgrading_from_version_prior_or_equal_to 2.3.2~ynh1; then ... +# example: if ynh_app_upgrading_from_version_before_or_equal_to 2.3.2~ynh1; then ... # # Requires YunoHost version 11.2 or higher. -ynh_app_upgrading_from_version_prior_or_equal_to() { +ynh_app_upgrading_from_version_before_or_equal_to() { local version=$1 [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" From 4dc59049ef87fb5addd6aef6f3ff6c50d9de491c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 19:01:14 +0200 Subject: [PATCH 051/146] helpers2.1: ynh_get_debian_release -> $YNH_DEBIAN_VERSION --- helpers/helpers.v2.1.d/mongodb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 62acc81e4..0881ed9af 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -224,7 +224,7 @@ ynh_install_mongo() { [[ -n "${mongo_version:-}" ]] || ynh_die --message="\$mongo_version should be defined prior to calling ynh_install_mongo" ynh_print_info --message="Installing MongoDB Community Edition ..." - local mongo_debian_release=$(ynh_get_debian_release) + local mongo_debian_release=$YNH_DEBIAN_VERSION if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." From 50eea8fc146cfa30456be722830312b28ce6f9c1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 22:47:29 +0200 Subject: [PATCH 052/146] helpers2.1: reintroduce the old ynh_restore as ynh_restore_everything because some apps are using it @_@ --- helpers/helpers.v2.1.d/backup | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 0f34e9bc9..fb9f239e4 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -225,6 +225,24 @@ ynh_restore() { fi } +# Restore all files that were previously backuped in an app backup script +# +# usage: ynh_restore_everything +# +# Requires YunoHost version 2.6.4 or higher. +ynh_restore_everything() { + # Deduce the relative path of $YNH_CWD + local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}" + REL_DIR="${REL_DIR%/}/" + + # For each destination path begining by $REL_DIR + cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \ + | while read line; do + local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)") + ynh_restore --target="$ARCHIVE_PATH" + done +} + # Calculate and store a file checksum into the app settings # # usage: ynh_store_file_checksum --file=file From 480366d5a1f3fab144bde148cab9ea40bc53bf66 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 10 Jun 2024 22:48:04 +0200 Subject: [PATCH 053/146] helpers2.1: in fact, let's use positional args for ynh_safe_rm because having --target everywhere is boring as hell --- helpers/helpers.v2.1.d/apt | 6 +++--- helpers/helpers.v2.1.d/backup | 2 +- helpers/helpers.v2.1.d/config | 2 +- helpers/helpers.v2.1.d/fail2ban | 4 ++-- helpers/helpers.v2.1.d/logging | 2 +- helpers/helpers.v2.1.d/mongodb | 4 ++-- helpers/helpers.v2.1.d/nginx | 4 ++-- helpers/helpers.v2.1.d/nodejs | 4 ++-- helpers/helpers.v2.1.d/php | 4 ++-- helpers/helpers.v2.1.d/ruby | 6 +++--- helpers/helpers.v2.1.d/systemd | 4 ++-- helpers/helpers.v2.1.d/utils | 17 ++++++----------- 12 files changed, 27 insertions(+), 32 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 24e5b5f99..fcd01b5be 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -481,10 +481,10 @@ EOF # Requires YunoHost version 3.8.1 or higher. ynh_remove_extra_repo() { - ynh_safe_rm --target="/etc/apt/sources.list.d/$app.list" - ynh_safe_rm --target="/etc/apt/preferences.d/$app" + ynh_safe_rm "/etc/apt/sources.list.d/$app.list" + ynh_safe_rm "/etc/apt/preferences.d/$app" if [ -e /etc/apt/trusted.gpg.d/$app.gpg ]; then - ynh_safe_rm --target="/etc/apt/trusted.gpg.d/$app.gpg" + ynh_safe_rm "/etc/apt/trusted.gpg.d/$app.gpg" fi ynh_package_update } diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index fb9f239e4..2a024fa8f 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -205,7 +205,7 @@ ynh_restore() { mkdir --parents "$(dirname "$backup_file")" mv "${target}" "$backup_file" # Move the current file or directory else - ynh_safe_rm --target=${target} + ynh_safe_rm "${target}" fi fi diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index 9e0127d36..317a2ce02 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -76,7 +76,7 @@ _ynh_app_config_apply_one() { local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]]; then ynh_backup_if_checksum_is_different --file="$bind_file" - ynh_safe_rm --target="$bind_file" + ynh_safe_rm "$bind_file" ynh_delete_file_checksum --file="$bind_file" ynh_print_info --message="File '$bind_file' removed" else diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index 580db74c9..cfc768e13 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -120,7 +120,7 @@ ignoreregex = # # Requires YunoHost version 3.5.0 or higher. ynh_remove_fail2ban_config() { - ynh_safe_rm --target="/etc/fail2ban/jail.d/$app.conf" - ynh_safe_rm --target="/etc/fail2ban/filter.d/$app.conf" + ynh_safe_rm "/etc/fail2ban/jail.d/$app.conf" + ynh_safe_rm "/etc/fail2ban/filter.d/$app.conf" ynh_systemd_action --service=fail2ban --action=reload } diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index 4721b45dc..0842e5740 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -80,7 +80,7 @@ ynh_exec_and_print_stderr_only_if_error() { "$@" 2> "$logfile" || rc="$?" if (( rc != 0 )); then ynh_exec_warn cat "$logfile" - ynh_safe_rm --target="$logfile" + ynh_safe_rm "$logfile" return "$rc" fi } diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 0881ed9af..c6750855d 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -265,7 +265,7 @@ ynh_remove_mongo() { mongodb_servicename=mongod # Remove the mongodb service yunohost service remove $mongodb_servicename - ynh_safe_rm --target="/var/lib/mongodb" - ynh_safe_rm --target="/var/log/mongodb" + ynh_safe_rm "/var/lib/mongodb" + ynh_safe_rm "/var/log/mongodb" fi } diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index 50b5a1a17..3a931fca8 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -39,7 +39,7 @@ ynh_add_nginx_config() { # # Requires YunoHost version 2.7.2 or higher. ynh_remove_nginx_config() { - ynh_safe_rm --target="/etc/nginx/conf.d/$domain.d/$app.conf" + ynh_safe_rm "/etc/nginx/conf.d/$domain.d/$app.conf" ynh_systemd_action --service=nginx --action=reload } @@ -58,7 +58,7 @@ ynh_change_url_nginx_config() { local old_nginx_conf_path=/etc/nginx/conf.d/$old_domain.d/$app.conf ynh_backup_if_checksum_is_different --file="$old_nginx_conf_path" ynh_delete_file_checksum --file="$old_nginx_conf_path" - ynh_safe_rm --target="$old_nginx_conf_path" + ynh_safe_rm "$old_nginx_conf_path" # Regen the nginx conf ynh_add_nginx_config diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index c8dfb94c7..a475b3cd2 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -172,8 +172,8 @@ ynh_remove_nodejs() { # If no other app uses n, remove n if [ ! -s "$n_install_dir/ynh_app_version" ]; then - ynh_safe_rm --target="$n_install_dir" - ynh_safe_rm --target="/usr/local/n" + ynh_safe_rm "$n_install_dir" + ynh_safe_rm "/usr/local/n" sed --in-place "/N_PREFIX/d" /root/.bashrc rm --force /etc/cron.daily/node_update fi diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 41097956e..65278dfa2 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -127,7 +127,7 @@ pm.process_idle_timeout = 10s # Validate that the new php conf doesn't break php-fpm entirely if ! php-fpm${phpversion} --test 2>/dev/null; then php-fpm${phpversion} --test || true - ynh_safe_rm --target="$finalphpconf" + ynh_safe_rm "$finalphpconf" ynh_die --message="The new configuration broke php-fpm?" fi ynh_systemd_action --service=$fpm_service --action=reload @@ -142,7 +142,7 @@ ynh_remove_fpm_config() { local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local fpm_service=$(ynh_app_setting_get --key=fpm_service) - ynh_safe_rm --target="$fpm_config_dir/pool.d/$app.conf" + ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" ynh_systemd_action --service=$fpm_service --action=reload } diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index eb8f8dc37..0259f270a 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -115,7 +115,7 @@ ynh_install_ruby () { else ynh_print_info --message="Reinstalling rbenv..." cd .. - ynh_safe_rm --target=$rbenv_install_dir + ynh_safe_rm $rbenv_install_dir mkdir -p $rbenv_install_dir cd $rbenv_install_dir git init -q @@ -284,8 +284,8 @@ ynh_cleanup_ruby () { then # Remove rbenv environment configuration ynh_print_info --message="Removing rbenv" - ynh_safe_rm --target="$rbenv_install_dir" - ynh_safe_rm --target="/etc/profile.d/rbenv.sh" + ynh_safe_rm "$rbenv_install_dir" + ynh_safe_rm "/etc/profile.d/rbenv.sh" fi } diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index a40d4afb8..56eb44428 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -46,7 +46,7 @@ ynh_remove_systemd_config() { if [ -e "$finalsystemdconf" ]; then ynh_systemd_action --service=$service --action=stop systemctl disable $service --quiet - ynh_safe_rm --target="$finalsystemdconf" + ynh_safe_rm "$finalsystemdconf" systemctl daemon-reload fi } @@ -181,6 +181,6 @@ ynh_clean_check_starting() { kill -SIGTERM $pid_tail 2>&1 fi if [ -n "${templog:-}" ]; then - ynh_safe_rm --target="$templog" 2>&1 + ynh_safe_rm "$templog" 2>&1 fi } diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 3901f04b8..abb34d650 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -259,7 +259,7 @@ ynh_setup_source() { fi if [ "$full_replace" -eq 1 ]; then - ynh_safe_rm --target="$dest_dir" + ynh_safe_rm "$dest_dir" fi # Extract source into the app dir @@ -285,11 +285,11 @@ ynh_setup_source() { local tmp_dir=$(mktemp --directory) unzip -quo $src_filename -d "$tmp_dir" cp --archive $tmp_dir/*/. "$dest_dir" - ynh_safe_rm --target="$tmp_dir" + ynh_safe_rm "$tmp_dir" else unzip -quo $src_filename -d "$dest_dir" fi - ynh_safe_rm --target="$src_filename" + ynh_safe_rm "$src_filename" else local strip="" if [ "$src_in_subdir" != "false" ]; then @@ -305,7 +305,7 @@ ynh_setup_source() { else ynh_die --message="Archive format unrecognized." fi - ynh_safe_rm --target="$src_filename" + ynh_safe_rm "$src_filename" fi # Apply patches @@ -429,16 +429,11 @@ _acceptable_path_to_delete() { # Remove a file or a directory securely # -# usage: ynh_safe_rm --target=path_to_remove -# | arg: -t, --target= - File or directory to remove +# usage: ynh_safe_rm path_to_remove # # Requires YunoHost version 2.6.4 or higher. ynh_safe_rm() { - # ============ Argument parsing ============= - local -A args_array=([t]=target=) - local target - ynh_handle_getopts_args "$@" - # =========================================== + local target="$1" set +o xtrace # set +x if [ $# -ge 2 ]; then From c5815fb1ef8966af4bd27cfc71dbab9a4044279f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 01:07:03 +0200 Subject: [PATCH 054/146] helpers2.1: simplify the logrotate mess: rename to ynh_add/remove_logrotate_config, use positional args, --specific_user is useless, we just need to make sure the parent dir has no +w on group... --- helpers/helpers.v2.1.d/logrotate | 36 ++++++++++---------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index 84f305195..c81e43d0c 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -2,26 +2,19 @@ FIRST_CALL_TO_LOGROTATE="true" -# Use logrotate to manage the logfile +# Add a logrotate configuration to manage log files / log directory # -# usage: ynh_use_logrotate [--logfile=/log/file] [--specific_user=user/group] -# | arg: -l, --logfile= - absolute path of logfile -# | arg: -u, --specific_user= - run logrotate as the specified user and group. If not specified logrotate is runned as root. +# usage: ynh_add_logrotate_config [/path/to/log/file/or/folder] # -# If no `--logfile` is provided, `/var/log/$app` will be used as default. -# `logfile` can point to a directory or a file. +# If not argument is provided, `/var/log/$app/*.log` is used as default. +# +# The configuration is autogenerated by YunoHost +# (ie it doesnt come from a specific app template like nginx or systemd conf) # # Requires YunoHost version 2.6.4 or higher. -ynh_use_logrotate() { +ynh_add_logrotate_config() { - # ============ Argument parsing ============= - local -A args_array=([l]=logfile= [u]=specific_user=) - local logfile - local specific_user - ynh_handle_getopts_args "$@" - logfile="${logfile:-}" - specific_user="${specific_user:-}" - # =========================================== + logfile="$1" set -o noglob if [[ -z "$logfile" ]]; then @@ -34,13 +27,10 @@ ynh_use_logrotate() { for stuff in $logfile do mkdir --parents $(dirname "$stuff") + # Make sure the permissions of the parent dir are correct (otherwise the config file could be ignored and the corresponding logs never rotated) + chmod 750 $(dirname "$stuff") done - local su_directive="" - if [[ -n "$specific_user" ]]; then - su_directive="su ${specific_user%/*} ${specific_user#*/}" - fi - local tempconf="$(mktemp)" cat << EOF >$tempconf $logfile { @@ -60,7 +50,6 @@ $logfile { notifempty # Keep old logs in the same dir noolddir - $su_directive } EOF @@ -73,10 +62,7 @@ EOF FIRST_CALL_TO_LOGROTATE="false" - # Make sure permissions are correct (otherwise the config file could be ignored and the corresponding logs never rotated) chmod 644 "/etc/logrotate.d/$app" - mkdir -p "/var/log/$app" - chmod 750 "/var/log/$app" } # Remove the app's logrotate config. @@ -84,7 +70,7 @@ EOF # usage: ynh_remove_logrotate # # Requires YunoHost version 2.6.4 or higher. -ynh_remove_logrotate() { +ynh_remove_logrotate_config() { if [ -e "/etc/logrotate.d/$app" ]; then rm "/etc/logrotate.d/$app" fi From 0c319cbeba2e23fdb774caa4d0e6fd6876c581a8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 02:13:28 +0200 Subject: [PATCH 055/146] helpers2.1: further simplify ynh_backup/restore: use a single positional arg. --is_big behavior is replaced by checking if the path is $data_dir (or a child) or /var/log/$app. --not_mandatory for restore is implied using the same check, or should be replaced by || true on the package side for the few cases where it's related to other stuff than the data dir or log dir --- helpers/helpers.v2.1.d/backup | 146 ++++++++++++---------------------- 1 file changed, 52 insertions(+), 94 deletions(-) diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 2a024fa8f..642c66e65 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -4,85 +4,48 @@ CAN_BIND=${CAN_BIND:-1} # Add a file or a directory to the list of paths to backup # -# usage: ynh_backup --target=/path/to/stuff [--is_big] [--not_mandatory] -# | arg: -s, --target= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. -# | arg: -b, --is_big - Indicate data are big (mail, video, image ...) -# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it. +# usage: ynh_backup /path/to/stuff # -# This helper can be used both in a system backup hook, and in an app backup script +# NB : note that this helper does *NOT* perform any copy in itself, it only +# declares stuff to be backuped via a CSV which is later picked up by the core # -# `ynh_backup` writes `target` and the corresponding path inside the archive (dest_path) into a CSV file, and it -# creates the parent destination directory +# NB 2 : there is a specific behavior for $data_dir (or childs of $data_dir) and +# /var/log/$app which are *NOT* backedup during safety-backup-before-upgrade, +# OR if the setting "do_not_backup_data" is equals 1 for that app # -# Example in the context of a wordpress app : -# ``` -# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" -# # => This line will be added into CSV file -# # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf" +# The rationale is that these directories are usually too heavy to be integrated in every backup +# (think for example about Nextcloud with quite a lot of data, or an app with a lot of media files...) # -# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf" -# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" +# This is coupled to the fact that $data_dir and the log dir won't be (and +# should NOT) be deleted during remove, unless --purge is used. Hence, if the +# upgrade fails and the script is removed prior to restoring the backup, the +# data/logs are not destroyed. # -# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/" -# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" -# -# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf" -# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf" -# -# #Deprecated usages (maintained for retro-compatibility) -# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf" -# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" -# -# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/" -# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" -# -# ``` -# -# How to use `--is_big`: -# -# `--is_big` is used to specify that this part of the backup can be quite huge. -# So, you don't want that your package does backup that part during ynh_backup_before_upgrade. -# In the same way, an user may doesn't want to backup this big part of the app for -# each of his backup. And so handle that part differently. -# -# As this part of your backup may not be done, your restore script has to handle it. -# In your restore script, use `--not_mandatory` with `ynh_restore` -# As well in your remove script, you should not remove those data ! Or an user may end up with -# a failed upgrade restoring an app without data anymore ! -# -# To have the benefit of `--is_big` while doing a backup, you can whether set the environement -# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect -# only that backup command. -# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect -# all backups for this app until the setting is removed. -# -# Requires YunoHost version 2.4.0 or higher. -# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory` ynh_backup() { - # TODO find a way to avoid injection by file strange naming ! - # ============ Argument parsing ============= - local -A args_array=([t]=target= [b]=is_big [m]=not_mandatory) - local target - local is_big - local not_mandatory - ynh_handle_getopts_args "$@" - is_big="${is_big:-0}" - not_mandatory="${not_mandatory:-0}" - # =========================================== + local target="$1" + local is_data=false - BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} - test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --key=do_not_backup_data) + # If the path starts with /var/log/$app or $data_dir + if ([[ -n "${app:-}" ]] && [[ "$target" == "/var/log/$app*" ]]) || ([[ -n "${data_dir:-}" ]] && [[ "$target" == "$data_dir*" ]]) + then + is_data=true + fi + + if [[ -n "${app:-}" ]] + then + local do_not_backup_data=$(ynh_app_setting_get --key=do_not_backup_data) + fi # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items - if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then + if [[ "$is_data" == true ]] && ([[ ${do_not_backup_data:-0} -eq 1 ]] || [[ ${BACKUP_CORE_ONLY:-0} -eq 1 ]]); then if [ $BACKUP_CORE_ONLY -eq 1 ]; then ynh_print_info --message="$target will not be saved, because 'BACKUP_CORE_ONLY' is set." else ynh_print_info --message="$target will not be saved, because 'do_not_backup_data' is set." fi - return 0 + return 1 fi # ============================================================================== @@ -90,12 +53,8 @@ ynh_backup() { # ============================================================================== # Be sure the source path is not empty if [ ! -e "$target" ]; then - ynh_print_warn --message="Source path '${src_path}' does not exist" - if [ "$not_mandatory" == "0" ]; then - return 1 - else - return 0 - fi + ynh_print_warn --message="File or folder '${target}' to be backed up does not exist" + return 1 fi # Transform the source path as an absolute path @@ -154,46 +113,45 @@ with open(sys.argv[1], 'r') as backup_file: return $? } -# Restore a file or a directory +# Restore a file or a directory from the backup archive # -# usage: ynh_restore --target=/path/to/stuff [--not_mandatory] -# | arg: -t, --target= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive -# | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it. -# -# Use the registered path in backup_list by ynh_backup to restore the file at the right place. +# usage: ynh_restore /path/to/stuff # # examples: -# ynh_restore -t "/etc/nginx/conf.d/$domain.d/$app.conf" -# # You can also use relative paths: -# ynh_restore -t "conf/nginx.conf" +# ynh_restore "/etc/nginx/conf.d/$domain.d/$app.conf" # -# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in -# `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed. +# If the file or dir to be restored already exists on the system and is lighter +# than 500 Mo, it is backed up in `/var/cache/yunohost/appconfbackup/`. +# Otherwise, the existing file or dir is removed. # # if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into # `/etc/nginx/conf.d/$domain.d/$app.conf` -# if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into +# otheriwse, search for a match in the csv (eg: conf/nginx.conf) and restore it into # `/etc/nginx/conf.d/$domain.d/$app.conf` # # Requires YunoHost version 2.6.4 or higher. -# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory ynh_restore() { - # ============ Argument parsing ============= - local -A args_array=([t]=target= [m]=not_mandatory) - local target - local not_mandatory - ynh_handle_getopts_args "$@" - target="/${target#/}" - not_mandatory="${not_mandatory:-0}" - # =========================================== + target="$1" local archive_path="$YNH_CWD${target}" + + # If the path starts with /var/log/$app or $data_dir + local is_data=false + if ([[ -n "${app:-}" ]] && [[ "$target" == "/var/log/$app*" ]]) || ([[ -n "${data_dir:-}" ]] && [[ "$target" == "$data_dir*" ]]) + then + is_data=true + fi + # If archive_path doesn't exist, search for a corresponding path in CSV if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then - if [ "$not_mandatory" == "0" ]; then - archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$target\")" - else + if [[ "$is_data" == true ]] + then + ynh_print_info --message="Skipping $target which doesn't exists in the archive, probably because restoring from a safety-backup-before-upgrade" + # Assume it's not a big deal, we may be restoring a safety-backup-before-upgrade which doesnt contain those return 0 + else + # (get_archive_path will raise an exception if no match found) + archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$target\")" fi fi @@ -239,7 +197,7 @@ ynh_restore_everything() { cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \ | while read line; do local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)") - ynh_restore --target="$ARCHIVE_PATH" + ynh_restore "$ARCHIVE_PATH" done } From 8c376c2ae454111834dcf3ef98a615e4a57afec9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 02:28:31 +0200 Subject: [PATCH 056/146] apps: remove /var/log/$app during app_remove if --purge is used --- src/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/app.py b/src/app.py index 25dce4104..3051fbbf2 100644 --- a/src/app.py +++ b/src/app.py @@ -1486,6 +1486,9 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): for permission_name in user_permission_list(apps=[app])["permissions"].keys(): permission_delete(permission_name, force=True, sync_perm=False) + if purge and os.path.exists(f"/var/log/{app}"): + shutil.rmtree(f"/var/log/{app}") + if os.path.exists(app_setting_path): shutil.rmtree(app_setting_path) From eb8ce2088b9ad0c1ba132720e73f30ffccb014ce Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 03:10:24 +0200 Subject: [PATCH 057/146] helpers2.1: go: add --quiet to goenv install + redirect stderr to stdout such that people don't slap an ynh_exec_warn_less in front of ynh_install_go x_x --- helpers/helpers.v2.1.d/go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 98def7f3f..ba729314f 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -153,7 +153,7 @@ ynh_install_go () { # Install the requested version of Go local final_go_version=$(goenv latest --print "$go_version") ynh_print_info --message="Installation of Go-$final_go_version" - goenv install --skip-existing "$final_go_version" + goenv install --quiet --skip-existing "$final_go_version" 2>&1 # Store go_version into the config of this app ynh_app_setting_set --app="$app" --key="go_version" --value="$final_go_version" From fca26ead788f64d5bcba90d0facfedc9bce64e32 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 03:16:03 +0200 Subject: [PATCH 058/146] helpers2.1: fix unecessary warnings + print_info in ruby, to also hopefully stop people from slapping an ynh_exec_warn_less in front of ynh_install_ruby --- helpers/helpers.v2.1.d/ruby | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 0259f270a..25f203a94 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -109,11 +109,11 @@ ynh_install_ruby () { if [ -n "$rbenv" ]; then pushd "${rbenv%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/rbenv/rbenv.git"; then - ynh_print_info --message="Updating rbenv..." + echo "Updating rbenv..." git pull -q --tags origin master ynh_ruby_try_bash_extension else - ynh_print_info --message="Reinstalling rbenv..." + echo "Reinstalling rbenv..." cd .. ynh_safe_rm $rbenv_install_dir mkdir -p $rbenv_install_dir @@ -126,7 +126,7 @@ ynh_install_ruby () { fi popd else - ynh_print_info --message="Installing rbenv..." + echo "Installing rbenv..." pushd $rbenv_install_dir git init -q git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 @@ -142,12 +142,12 @@ ynh_install_ruby () { if [ -n "$ruby_build" ]; then pushd "${ruby_build%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/rbenv/ruby-build.git"; then - ynh_print_info --message="Updating ruby-build..." + echo "Updating ruby-build..." git pull -q origin master fi popd else - ynh_print_info --message="Installing ruby-build..." + echo "Installing ruby-build..." git clone -q https://github.com/rbenv/ruby-build.git "${rbenv_install_dir}/plugins/ruby-build" fi @@ -155,12 +155,12 @@ ynh_install_ruby () { if [ -n "$rbenv_alias" ]; then pushd "${rbenv_alias%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/tpope/rbenv-aliases.git"; then - ynh_print_info --message="Updating rbenv-aliases..." + echo "Updating rbenv-aliases..." git pull -q origin master fi popd else - ynh_print_info --message="Installing rbenv-aliases..." + echo "Installing rbenv-aliases..." git clone -q https://github.com/tpope/rbenv-aliases.git "${rbenv_install_dir}/plugins/rbenv-aliase" fi @@ -168,12 +168,12 @@ ynh_install_ruby () { if [ -n "$rbenv_latest" ]; then pushd "${rbenv_latest%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then - ynh_print_info --message="Updating xxenv-latest..." + echo "Updating xxenv-latest..." git pull -q origin master fi popd else - ynh_print_info --message="Installing xxenv-latest..." + echo "Installing xxenv-latest..." git clone -q https://github.com/momo-lab/xxenv-latest.git "${rbenv_install_dir}/plugins/xxenv-latest" fi @@ -194,7 +194,7 @@ ynh_install_ruby () { if ! [ -n "$final_ruby_version" ]; then final_ruby_version=$ruby_version fi - ynh_print_info --message="Installing Ruby $final_ruby_version" + echo "Installing Ruby $final_ruby_version" RUBY_CONFIGURE_OPTS="--disable-install-doc --with-jemalloc" MAKE_OPTS="-j2" rbenv install --skip-existing $final_ruby_version > /dev/null 2>&1 # Store ruby_version into the config of this app @@ -274,7 +274,7 @@ ynh_cleanup_ruby () { do if ! echo ${required_ruby_versions} | grep -q "${installed_ruby_version}" then - ynh_print_info --message="Removing Ruby-$installed_ruby_version" + echo "Removing Ruby-$installed_ruby_version" $rbenv_install_dir/bin/rbenv uninstall --force $installed_ruby_version fi done @@ -283,7 +283,7 @@ ynh_cleanup_ruby () { if [[ -z "$required_ruby_versions" ]] then # Remove rbenv environment configuration - ynh_print_info --message="Removing rbenv" + echo "Removing rbenv" ynh_safe_rm "$rbenv_install_dir" ynh_safe_rm "/etc/profile.d/rbenv.sh" fi @@ -291,7 +291,7 @@ ynh_cleanup_ruby () { ynh_ruby_try_bash_extension() { if [ -x src/configure ]; then - src/configure && make -C src || { + src/configure && make -C src 2>&1 || { ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." } fi From b4b420d69439b4142b3d978d53df5389a6b7557c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 16:20:28 +0200 Subject: [PATCH 059/146] helpers2.1: ynh_exec_as $app -> ynh_exec_as_app (with -E option in sudo) --- helpers/helpers.v2.1.d/user | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index c971a415b..0510fd721 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -172,13 +172,11 @@ ynh_system_user_delete() { fi } -# Execute a command as another user +# Execute a command after sudoing as $app # -# usage: ynh_exec_as $USER COMMAND [ARG ...] +# Note that exported bash env variables are kept (using -E option of sudo) # -# Requires YunoHost version 4.1.7 or higher. -ynh_exec_as() { - local user=$1 - shift 1 - sudo -u "$user" "$@" +# usage: ynh_exec_as_app COMMAND [ARG ...] +ynh_exec_as_app() { + sudo -E -u"$app" "$@" } From a8cd94d3db9db86b4a7830888a993ebddcc77bd7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 18:20:22 +0200 Subject: [PATCH 060/146] helpers2.1: rework the nodejs/ruby/go mess: export the appropriate extended PATH to be able to call npm/node/ruby/gem/go directly (no ynh_foo anymore). No more ynh_use_foo either, it's just automatically called at the end of ynh_install_foo. Also rework the ynh_foo_load_PATH variable to PATH_with_foo, such that we shall write Enviroment="PATH=__PATH_WITH_FOO__" in the systemd conf to be more explicit --- helpers/helpers.v2.1.d/go | 70 +++++++------------------------ helpers/helpers.v2.1.d/nodejs | 79 +++++++---------------------------- helpers/helpers.v2.1.d/ruby | 72 ++++++++----------------------- 3 files changed, 47 insertions(+), 174 deletions(-) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index ba729314f..12c089744 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -13,67 +13,25 @@ go_version_path="$goenv_install_dir/versions" # goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable. export GOENV_ROOT="$goenv_install_dir" -# Load the version of Go for an app, and set variables. -# -# ynh_use_go has to be used in any app scripts before using Go for the first time. -# This helper will provide alias and variables to use in your scripts. -# -# To use gem or Go, use the alias `ynh_gem` and `ynh_go` -# Those alias will use the correct version installed for the app -# For example: use `ynh_gem install` instead of `gem install` -# -# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_go` -# And propagate $PATH to sudo with $ynh_go_load_path -# Exemple: `ynh_exec_as $app $ynh_go_load_path $ynh_gem install` -# -# $PATH contains the path of the requested version of Go. -# However, $PATH is duplicated into $go_path to outlast any manipulation of $PATH -# You can use the variable `$ynh_go_load_path` to quickly load your Go version -# in $PATH for an usage into a separate script. -# Exemple: `$ynh_go_load_path $install_dir/script_that_use_gem.sh` -# -# -# Finally, to start a Go service with the correct version, 2 solutions -# Either the app is dependent of Go or gem, but does not called it directly. -# In such situation, you need to load PATH -# `Environment="__YNH_GO_LOAD_PATH__"` -# `ExecStart=__INSTALL_DIR__/my_app` -# You will replace __YNH_GO_LOAD_PATH__ with $ynh_go_load_path -# -# Or Go start the app directly, then you don't need to load the PATH variable -# `ExecStart=__YNH_GO__ my_app run` -# You will replace __YNH_GO__ with $ynh_go -# -# -# one other variable is also available -# - $go_path: The absolute path to Go binaries for the chosen version. -# -# usage: ynh_use_go -# -# Requires YunoHost version 3.2.2 or higher. -ynh_use_go () { +_ynh_load_go_in_path_and_other_tweaks() { - [[ -n "${go_version:-}" ]] || ynh_die --message="\$go_version should be defined prior to calling ynh_use_go" + # Get the absolute path of this version of go + local go_path="$go_version_path/$app/bin" - # Get the absolute path of this version of Go - go_path="$go_version_path/$go_version/bin" - - # Allow alias to be used into bash script - shopt -s expand_aliases - - # Create an alias for the specific version of Go and a variable as fallback - ynh_go="$go_path/go" - alias ynh_go="$ynh_go" - - # Load the path of this version of Go in $PATH + # Load the path of this version of go in $PATH if [[ :$PATH: != *":$go_path"* ]]; then PATH="$go_path:$PATH" fi - # Create an alias to easily load the PATH - ynh_go_load_path="PATH=$PATH" - # Sets the local application-specific Go version - pushd $install_dir + # Export PATH such that it's available through sudo -E / ynh_exec_as $app + export PATH + + # This is in full lowercase such that it gets replaced in templates + path_with_go="$PATH" + PATH_with_go="$PATH" + + # Sets the local application-specific go version + pushd ${install_dir} $goenv_install_dir/bin/goenv local $go_version popd } @@ -171,6 +129,8 @@ eval \"\$(goenv init -)\" # Load the environment eval "$(goenv init -)" + + _ynh_load_go_in_path_and_other_tweaks } # Remove the version of Go used by the app. diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index a475b3cd2..0b8788409 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -5,75 +5,23 @@ node_version_path="$n_install_dir/n/versions/node" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. export N_PREFIX="$n_install_dir" -# Load the version of node for an app, and set variables. -# -# usage: ynh_use_nodejs -# -# `ynh_use_nodejs` has to be used in any app scripts before using node for the first time. -# This helper will provide alias and variables to use in your scripts. -# -# To use npm or node, use the alias `ynh_npm` and `ynh_node`. -# -# Those alias will use the correct version installed for the app. -# For example: use `ynh_npm install` instead of `npm install` -# -# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_npm` and `$ynh_node` -# And propagate $PATH to sudo with $ynh_node_load_PATH -# Exemple: `ynh_exec_as $app $ynh_node_load_PATH $ynh_npm install` -# -# $PATH contains the path of the requested version of node. -# However, $PATH is duplicated into $node_PATH to outlast any manipulation of `$PATH` -# You can use the variable `$ynh_node_load_PATH` to quickly load your node version -# in $PATH for an usage into a separate script. -# Exemple: $ynh_node_load_PATH $install_dir/script_that_use_npm.sh` -# -# -# Finally, to start a nodejs service with the correct version, 2 solutions -# Either the app is dependent of node or npm, but does not called it directly. -# In such situation, you need to load PATH : -# ``` -# Environment="__NODE_ENV_PATH__" -# ExecStart=__FINALPATH__/my_app -# ``` -# You will replace __NODE_ENV_PATH__ with $ynh_node_load_PATH. -# -# Or node start the app directly, then you don't need to load the PATH variable -# ``` -# ExecStart=__YNH_NODE__ my_app run -# ``` -# You will replace __YNH_NODE__ with $ynh_node -# -# -# 2 other variables are also available -# - $nodejs_path: The absolute path to node binaries for the chosen version. -# -# Requires YunoHost version 2.7.12 or higher. -ynh_use_nodejs() { - - [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_install_nodejs" +_ynh_load_nodejs_in_path_and_other_tweaks() { # Get the absolute path of this version of node - nodejs_path="$node_version_path/$nodejs_version/bin" - - # Allow alias to be used into bash script - shopt -s expand_aliases - - # Create an alias for the specific version of node and a variable as fallback - ynh_node="$nodejs_path/node" - alias ynh_node="$ynh_node" - # And npm - ynh_npm="$nodejs_path/npm" - alias ynh_npm="$ynh_npm" + local nodejs_path="$node_version_path/$nodejs_version/bin" # Load the path of this version of node in $PATH if [[ :$PATH: != *":$nodejs_path"* ]]; then PATH="$nodejs_path:$PATH" fi - node_PATH="$PATH" - # Create an alias to easily load the PATH - ynh_node_load_PATH="PATH=$node_PATH" - # Same var but in lower case to be compatible with ynh_replace_vars... - ynh_node_load_path="PATH=$node_PATH" + + # Export PATH such that it's available through sudo -E / ynh_exec_as $app + export PATH + + # This is in full lowercase such that it gets replaced in templates + path_with_nodejs="$PATH" + PATH_with_nodejs="$PATH" + # Prevent yet another Node and Corepack madness, with Corepack wanting the user to confirm download of Yarn export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 } @@ -87,7 +35,10 @@ ynh_use_nodejs() { # `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use. # That's how it changes the version # -# Refer to `ynh_use_nodejs` for more information about available commands and variables +# Adds the appropriate, specific version of nodejs to the PATH variable (which +# is also exported, to ease the use of ynh_exec_as_app). Also define variable +# PATH_with_nodejs to be used in the systemd config +# (Environment="PATH=__PATH_WITH_NODEJS__") # # Requires YunoHost version 2.7.12 or higher. ynh_install_nodejs() { @@ -146,7 +97,7 @@ ynh_install_nodejs() { # Build the update script and set the cronjob ynh_cron_upgrade_node - ynh_use_nodejs + _ynh_load_nodejs_in_path_and_other_tweaks } # Remove the version of node used by the app. diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 25f203a94..acc36ac66 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -7,67 +7,22 @@ ruby_version_path="$rbenv_install_dir/versions" export RBENV_ROOT="$rbenv_install_dir" export rbenv_root="$rbenv_install_dir" -# Load the version of Ruby for an app, and set variables. -# -# ynh_use_ruby has to be used in any app scripts before using Ruby for the first time. -# This helper will provide alias and variables to use in your scripts. -# -# To use gem or Ruby, use the alias `ynh_gem` and `ynh_ruby` -# Those alias will use the correct version installed for the app -# For example: use `ynh_gem install` instead of `gem install` -# -# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_ruby` -# And propagate $PATH to sudo with $ynh_ruby_load_path -# Exemple: `ynh_exec_as $app $ynh_ruby_load_path $ynh_gem install` -# -# $PATH contains the path of the requested version of Ruby. -# However, $PATH is duplicated into $ruby_path to outlast any manipulation of $PATH -# You can use the variable `$ynh_ruby_load_path` to quickly load your Ruby version -# in $PATH for an usage into a separate script. -# Exemple: $ynh_ruby_load_path $install_dir/script_that_use_gem.sh` -# -# -# Finally, to start a Ruby service with the correct version, 2 solutions -# Either the app is dependent of Ruby or gem, but does not called it directly. -# In such situation, you need to load PATH -# `Environment="__YNH_RUBY_LOAD_PATH__"` -# `ExecStart=__FINALPATH__/my_app` -# You will replace __YNH_RUBY_LOAD_PATH__ with $ynh_ruby_load_path -# -# Or Ruby start the app directly, then you don't need to load the PATH variable -# `ExecStart=__YNH_RUBY__ my_app run` -# You will replace __YNH_RUBY__ with $ynh_ruby -# -# -# one other variable is also available -# - $ruby_path: The absolute path to Ruby binaries for the chosen version. -# -# usage: ynh_use_ruby -# -# Requires YunoHost version 3.2.2 or higher. -ynh_use_ruby () { - - [[ -n "${ruby_version:-}" ]] || ynh_die --message="\$ruby_version should be defined prior to calling ynh_use_ruby" +_ynh_load_ruby_in_path_and_other_tweaks() { # Get the absolute path of this version of Ruby - ruby_path="$ruby_version_path/$app/bin" + local ruby_path="$ruby_version_path/$app/bin" - # Allow alias to be used into bash script - shopt -s expand_aliases - - # Create an alias for the specific version of Ruby and a variable as fallback - ynh_ruby="$ruby_path/ruby" - alias ynh_ruby="$ynh_ruby" - # And gem - ynh_gem="$ruby_path/gem" - alias ynh_gem="$ynh_gem" - - # Load the path of this version of Ruby in $PATH + # Load the path of this version of ruby in $PATH if [[ :$PATH: != *":$ruby_path"* ]]; then PATH="$ruby_path:$PATH" fi - # Create an alias to easily load the PATH - ynh_ruby_load_path="PATH=$PATH" + + # Export PATH such that it's available through sudo -E / ynh_exec_as $app + export PATH + + # This is in full lowercase such that it gets replaced in templates + path_with_ruby="$PATH" + PATH_with_ruby="$PATH" # Sets the local application-specific Ruby version pushd ${install_dir} @@ -89,6 +44,11 @@ ynh_use_ruby () { # # usage: ynh_install_ruby # +# Adds the appropriate, specific version of ruby to the PATH variable (which +# is also exported, to ease the use of ynh_exec_as_app). Also define variable +# PATH_with_ruby to be used in the systemd config +# (Environment="PATH=__PATH_WITH_RUBY__") +# # Requires YunoHost version 3.2.2 or higher. ynh_install_ruby () { @@ -222,6 +182,8 @@ eval \"\$(rbenv init -)\" # Load the environment eval "$(rbenv init -)" + + _ynh_load_ruby_in_path_and_other_variable_tweaks } # Remove the version of Ruby used by the app. From 66508d5fd6cd1310cf413daef5199ab7d8999e98 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 18:41:37 +0200 Subject: [PATCH 061/146] helpers2.1: ynh_exec_warn_less -> ynh_hide_warnings --- helpers/helpers.v2.1.d/logging | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index 0842e5740..9993739a4 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -52,15 +52,12 @@ ynh_print_warn() { echo -e "${message}" >&2 } -# Execute a command and force the result to be printed on stdout +# Execute a command and redirect stderr to stdout # -# usage: ynh_exec_warn_less your command and args +# usage: ynh_hide_warnings your command and args # | arg: command - command to execute # -# Note that you should NOT quote the command but only prefix it with ynh_exec_warn -# -# Requires YunoHost version 3.2.0 or higher. -ynh_exec_warn_less() { +ynh_hide_warnings() { # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 "$@" 2>&1 } @@ -79,7 +76,7 @@ ynh_exec_and_print_stderr_only_if_error() { # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 "$@" 2> "$logfile" || rc="$?" if (( rc != 0 )); then - ynh_exec_warn cat "$logfile" + cat "$logfile" >&2 ynh_safe_rm "$logfile" return "$rc" fi From 47b2a5695f3a1dfd399e8c9a4cc572dd19c74f5f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 19:27:33 +0200 Subject: [PATCH 062/146] helpers2.1: replace ynh_check_app_version_changed with a much simpler ynh_app_upstream_version_changed that can directly be used with 'if ynh_app_upstream_version_changed' instead of the current mess --- helpers/helpers.v2.1.d/utils | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index abb34d650..18b3d784f 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -476,23 +476,12 @@ ynh_app_upstream_version() { echo "${$YNH_APP_MANIFEST_VERSION/~ynh*/}" } -# Checks the app version to upgrade with the existing app version and returns: +# Return 0 if the "upstream" part of the version changed, or 1 otherwise (ie only the ~ynh suffix changed) # -# usage: ynh_check_app_version_changed -# | ret: `UPGRADE_APP` if the upstream version changed, `UPGRADE_PACKAGE` otherwise. -# -# This helper should be used to avoid an upgrade of an app, or the upstream part -# of it, when it's not needed -# -# Requires YunoHost version 3.5.0 or higher. -ynh_check_app_version_changed() { - local return_value=${YNH_APP_UPGRADE_TYPE} - - if [ "$return_value" == "UPGRADE_SAME" ] || [ "$return_value" == "DOWNGRADE" ]; then - return_value="UPGRADE_APP" - fi - - echo $return_value +# usage: if ynh_app_upstream_version_changed; then ... +ynh_app_upstream_version_changed() { + # "UPGRADE_PACKAGE" means only the ~ynh prefix changed + [[ "$YNH_APP_UPGRADE_TYPE" == "UPGRADE_PACKAGE" ]] && return 1 || return 0 } # Compare the current package version is strictly lower than another version given as an argument From 66f667e48a20aaf8694ef936f2ce5082254de1bf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jun 2024 22:55:30 +0200 Subject: [PATCH 063/146] helpers2.1: in ynh_setup_source, remove the underused sources/extra_files/sourceid/ mechanism (basically the only apps with these files manually cp stuff) + move the sources/patches/ mechanism to just patches/ --- helpers/helpers.v2.1.d/utils | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 18b3d784f..889075968 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -132,8 +132,7 @@ fi # - Uncompress the archive to `$dest_dir`. # - If `in_subdir` is true, the first level directory of the archive will be removed. # - If `in_subdir` is a numeric value, the N first level directories will be removed. -# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` -# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir +# - Patches named `patches/${src_id}-*.patch` will be applied to `$dest_dir` # # Requires YunoHost version 2.6.4 or higher. ynh_setup_source() { @@ -309,9 +308,10 @@ ynh_setup_source() { fi # Apply patches - if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then - local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) - if (($(find $patches_folder -type f -name "${source_id}-*.patch" 2>/dev/null | wc --lines) > "0")); then + if [ -d "$YNH_APP_BASEDIR/patches/" ]; then + local patches_folder=$(realpath $YNH_APP_BASEDIR/patches/) + # Check if any file matching the pattern exists, cf https://stackoverflow.com/a/34195247 + if compgen -G "$patches_folder/${source_id}-*.patch" >/dev/null; then pushd "$dest_dir" for p in $patches_folder/${source_id}-*.patch; do echo $p @@ -321,11 +321,6 @@ ynh_setup_source() { fi fi - # Add supplementary files - if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then - cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir" - fi - # Keep files to be backup/restored at the end of the helper # Assuming $dest_dir already exists if [ -n "$keep" ]; then From d501131b34d0ebd484d4b9a6c62affe1c4375bd3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 12 Jun 2024 15:53:44 +0200 Subject: [PATCH 064/146] helpers2.1: autorename phpversion to php_version for consistency with nodejs_version, ruby_version, ... --- helpers/helpers.v2.1.d/apps | 6 +++--- helpers/helpers.v2.1.d/apt | 12 ++++++------ helpers/helpers.v2.1.d/php | 36 +++++++++++++++++++++++------------- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps index 474f5e45f..a3ee183a9 100644 --- a/helpers/helpers.v2.1.d/apps +++ b/helpers/helpers.v2.1.d/apps @@ -173,11 +173,11 @@ ynh_spawn_app_shell() { # Force `php` to its intended version # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` - local phpversion=$(ynh_app_setting_get --key=phpversion) + local php_version=$(ynh_app_setting_get --key=php_version) local phpflags=$(ynh_app_setting_get --key=phpflags) - if [ -n "$phpversion" ] + if [ -n "$php_version" ] then - eval "php() { php${phpversion} ${phpflags} \"\$@\"; }" + eval "php() { php${php_version} ${phpflags} \"\$@\"; }" export -f php fi diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index fcd01b5be..3b8666078 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -266,10 +266,10 @@ ynh_install_app_dependencies() { dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common" - local old_phpversion=$(ynh_app_setting_get --key=phpversion) + local old_php_version=$(ynh_app_setting_get --key=php_version) # If the PHP version changed, remove the old fpm conf - if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$specific_php_version" ]; then + if [ -n "$old_php_version" ] && [ "$old_php_version" != "$specific_php_version" ]; then local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" @@ -279,8 +279,8 @@ ynh_install_app_dependencies() { ynh_remove_fpm_config fi fi - # Store phpversion into the config of this app - ynh_app_setting_set --key=phpversion --value=$specific_php_version + # Store php_version into the config of this app + ynh_app_setting_set --key=php_version --value=$specific_php_version # Set the default php version back as the default version for php-cli. if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION @@ -288,7 +288,7 @@ ynh_install_app_dependencies() { update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION fi elif grep --quiet 'php' <<< "$dependencies"; then - ynh_app_setting_set --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION + ynh_app_setting_set --key=php_version --value=$YNH_DEFAULT_PHP_VERSION fi local psql_installed="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)" @@ -339,7 +339,7 @@ EOF # # [packagingv1] # -# usage: ynh_add_app_dependencies --package=phpversion +# usage: ynh_add_app_dependencies --package=packagename # | arg: -p, --package= - Packages to add as dependencies for the app. # # Requires YunoHost version 3.8.1 or higher. diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 65278dfa2..2c0500eb5 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -1,9 +1,19 @@ #!/bin/bash +# (this is used in the apt helpers, big meh ...) readonly YNH_DEFAULT_PHP_VERSION=7.4 -# Declare the actual PHP version to use. -# A packager willing to use another version of PHP can override the variable into its _common.sh. -YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} + +# Legacy: auto-convert phpversion to php_version (for consistency with nodejs_version, ruby_version, ...) +if [[ -n "${app:-}" ]] && [[ -n "${phpversion:-}" ]] +then + if [[ -z "${php_version:-}" ]] + then + php_version=$phpversion + ynh_app_setting_set --key=php_version --value=$php_version + fi + ynh_app_setting_delete --key=phpversion + unset phpversion +fi # Create a dedicated PHP-FPM config # @@ -14,7 +24,7 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # The actual PHP configuration will be automatically generated, # and your extra_php-fpm.conf will be appended (typically contains PHP upload limits) # -# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf +# The resulting configuration will be deployed to the appropriate place, /etc/php/$php_version/fpm/pool.d/$app.conf # # Performance-related options in the PHP conf, such as : # pm.max_children, pm.start_servers, pm.min_spare_servers pm.max_spare_servers @@ -58,8 +68,8 @@ ynh_add_fpm_config() { # If the PHP version changed, remove the old fpm conf # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper) - local old_phpversion=$(ynh_app_setting_get --key=phpversion) - if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then + local old_php_version=$(ynh_app_setting_get --key=php_version) + if [ -n "$old_php_version" ] && [ "$old_php_version" != "$php_version" ]; then local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" @@ -70,15 +80,15 @@ ynh_add_fpm_config() { fi fi - local fpm_service="php${phpversion}-fpm" - local fpm_config_dir="/etc/php/$phpversion/fpm" + local fpm_service="php${php_version}-fpm" + local fpm_config_dir="/etc/php/$php_version/fpm" # Create the directory for FPM pools mkdir --parents "$fpm_config_dir/pool.d" ynh_app_setting_set --key=fpm_config_dir --value="$fpm_config_dir" ynh_app_setting_set --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --key=phpversion --value=$phpversion + ynh_app_setting_set --key=php_version --value=$php_version # Define the values to use for the configuration of PHP. ynh_get_scalable_phpfpm @@ -93,7 +103,7 @@ group = __PHPFPM_GROUP__ chdir = __INSTALL_DIR__ -listen = /var/run/php/php__PHPVERSION__-fpm-__APP__.sock +listen = /var/run/php/php__PHP_VERSION__-fpm-__APP__.sock listen.owner = www-data listen.group = www-data @@ -125,8 +135,8 @@ pm.process_idle_timeout = 10s ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" # Validate that the new php conf doesn't break php-fpm entirely - if ! php-fpm${phpversion} --test 2>/dev/null; then - php-fpm${phpversion} --test || true + if ! php-fpm${php_version} --test 2>/dev/null; then + php-fpm${php_version} --test || true ynh_safe_rm "$finalphpconf" ynh_die --message="The new configuration broke php-fpm?" fi @@ -298,7 +308,7 @@ ynh_composer_exec() { # =========================================== COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ - php${phpversion} "$workdir/composer.phar" $commands \ + php${php_version} "$workdir/composer.phar" $commands \ -d "$workdir" --no-interaction --no-ansi 2>&1 } From 50a4d08add362d8ef2c4cfea5f8e1c96be82b519 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 13 Jun 2024 01:00:52 +0200 Subject: [PATCH 065/146] helpers2.1: fixes after tests on the battlefield --- helpers/helpers | 2 +- helpers/helpers.v2.1.d/apt | 2 +- helpers/helpers.v2.1.d/php | 26 ++------------------------ helpers/helpers.v2.1.d/string | 2 +- helpers/helpers.v2.1.d/utils | 4 ++-- helpers/helpers.v2.1.d/vendor | 1 + 6 files changed, 8 insertions(+), 29 deletions(-) create mode 120000 helpers/helpers.v2.1.d/vendor diff --git a/helpers/helpers b/helpers/helpers index 01b6fa7e3..64f9322ae 100644 --- a/helpers/helpers +++ b/helpers/helpers @@ -11,7 +11,7 @@ set +x YNH_HELPERS_DIR="$SCRIPT_DIR/helpers.v${YNH_HELPERS_VERSION}.d" case "$YNH_HELPERS_VERSION" in - "1" | "2") + "1" | "2" | "2.1") readarray -t HELPERS < <(find -L "$YNH_HELPERS_DIR" -mindepth 1 -maxdepth 1 -type f) for helper in "${HELPERS[@]}"; do [ -r "$helper" ] && source "$helper" diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 3b8666078..fa35aa45c 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -234,7 +234,7 @@ ynh_install_app_dependencies() { dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" local dependencies=${dependencies//|/ | } - local version=$(ynh_read_manifest --manifest_key="version") + local version=$(ynh_read_manifest "version") if [ -z "${version}" ] || [ "$version" == "null" ]; then version="1.0" fi diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 2c0500eb5..37fe76785 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -217,7 +217,7 @@ ynh_get_scalable_phpfpm() { fi # Get the total of RAM available, except swap. - local max_ram=$(ynh_get_ram --total --ignore_swap) + local max_ram=$(ynh_get_ram --total) at_least_one() { # Do not allow value below 1 @@ -267,28 +267,6 @@ ynh_get_scalable_phpfpm() { php_max_spare_servers=0 php_start_servers=0 fi - - if [ $print -eq 1 ]; then - ynh_print_warn --message="Footprint=${footprint}Mb by pool." - ynh_print_warn --message="Process manager=$php_pm" - ynh_print_warn --message="Max RAM=${max_ram}Mb" - if [ "$php_pm" != "static" ]; then - ynh_print_warn --message="\nMax estimated footprint=$(($php_max_children * $footprint))" - ynh_print_warn --message="Min estimated footprint=$(($php_min_spare_servers * $footprint))" - fi - if [ "$php_pm" = "dynamic" ]; then - ynh_print_warn --message="Estimated average footprint=$(($php_max_spare_servers * $footprint))" - elif [ "$php_pm" = "static" ]; then - ynh_print_warn --message="Estimated footprint=$(($php_max_children * $footprint))" - fi - ynh_print_warn --message="\nRaw php-fpm values:" - ynh_print_warn --message="pm.max_children = $php_max_children" - if [ "$php_pm" = "dynamic" ]; then - ynh_print_warn --message="pm.start_servers = $php_start_servers" - ynh_print_warn --message="pm.min_spare_servers = $php_min_spare_servers" - ynh_print_warn --message="pm.max_spare_servers = $php_max_spare_servers" - fi - fi } # Execute a command with Composer @@ -342,7 +320,7 @@ ynh_install_composer() { # because local always return 0 ... local out # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) - out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir $composer_url 2>&1) \ + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir/composer.phar $composer_url 2>&1) \ || ynh_die --message="$out" # install dependencies diff --git a/helpers/helpers.v2.1.d/string b/helpers/helpers.v2.1.d/string index bacb95f93..8a0fbab93 100644 --- a/helpers/helpers.v2.1.d/string +++ b/helpers/helpers.v2.1.d/string @@ -38,7 +38,7 @@ ynh_string_random() { # Requires YunoHost version 2.6.4 or higher. ynh_replace() { # ============ Argument parsing ============= - local -A args_array=([m]=match= [r]=replaceg= [f]=file=) + local -A args_array=([m]=match= [r]=replace= [f]=file=) local match local replace local file diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 889075968..8326f644f 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -388,7 +388,7 @@ ynh_local_curl() { chmod 700 $cookiefile # Temporarily enable visitors if needed... - local visitors_enabled=$(ynh_permission_has_user "main" "visitors" && echo yes || echo no) + local visitors_enabled=$(ynh_permission_has_user --permission="main" --user="visitors" && echo yes || echo no) if [[ $visitors_enabled == "no" ]]; then ynh_permission_update --permission="main" --add="visitors" fi @@ -456,7 +456,7 @@ ynh_safe_rm() { # # Requires YunoHost version 3.5.0 or higher. ynh_read_manifest() { - cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".$manifest_key" --raw-output + cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq "$1" --raw-output } # Return the app upstream version, deduced from `$YNH_APP_MANIFEST_VERSION` and strippig the `~ynhX` part diff --git a/helpers/helpers.v2.1.d/vendor b/helpers/helpers.v2.1.d/vendor new file mode 120000 index 000000000..9c39cc9f8 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor @@ -0,0 +1 @@ +../vendor \ No newline at end of file From 3584e6a5b1fc7e5e6f167684dac3041677f95f04 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 13 Jun 2024 11:28:52 +0200 Subject: [PATCH 066/146] helpers2.1: remove legacy ynh_add_app_dependencies --- helpers/helpers.v2.1.d/apt | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index fa35aa45c..38eb03c13 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -335,25 +335,6 @@ EOF } -# Add dependencies to install with ynh_install_app_dependencies -# -# [packagingv1] -# -# usage: ynh_add_app_dependencies --package=packagename -# | arg: -p, --package= - Packages to add as dependencies for the app. -# -# Requires YunoHost version 3.8.1 or higher. -ynh_add_app_dependencies() { - # ============ Argument parsing ============= - local -A args_array=([p]=package=) - local package - ynh_handle_getopts_args "$@" - # =========================================== - - ynh_print_warn --message="Packagers: ynh_add_app_dependencies is deprecated and is now only an alias to ynh_install_app_dependencies" - ynh_install_app_dependencies "${package}" -} - # Remove fake package and its dependencies # # [packagingv1] From 800f93d12e587e81be9ecd7fdbeb11a79a77257f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 13 Jun 2024 12:39:05 +0200 Subject: [PATCH 067/146] helpers2.1: further simplify mysql/postgresql helper: no keyword arg, remove ynh_foo_setup_db and ynh_foo_remove_db (the other helpers are enough), replace ynh_foosql_connect_as/ynh_foosql_execute* with a single ynh_foosql_db_shell that reads stdin --- helpers/helpers.v1.d/mysql | 15 ++- helpers/helpers.v2.1.d/mysql | 192 ++++----------------------- helpers/helpers.v2.1.d/postgresql | 214 ++++-------------------------- src/utils/resources.py | 15 ++- 4 files changed, 76 insertions(+), 360 deletions(-) diff --git a/helpers/helpers.v1.d/mysql b/helpers/helpers.v1.d/mysql index c11f7989a..98e9a745e 100644 --- a/helpers/helpers.v1.d/mysql +++ b/helpers/helpers.v1.d/mysql @@ -174,6 +174,19 @@ ynh_mysql_user_exists() { fi } +# Check if a mysql database exists +# +# [internal] +# +# usage: ynh_mysql_database_exists database +# | arg: database - the database for which to check existence +# | exit: Return 1 if the database doesn't exist, 0 otherwise +# +ynh_mysql_database_exists() { + local database=$1 + mysqlshow | grep -q "^| $database " +} + # Drop a user # # [internal] @@ -236,7 +249,7 @@ ynh_mysql_remove_db() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if mysqlshow | grep -q "^| $db_name "; then + if ynh_mysql_database_exists "$db_name"; then ynh_mysql_drop_db $db_name else ynh_print_warn --message="Database $db_name not found" diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index e8cf9a69d..15559b2bc 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -1,76 +1,17 @@ #!/bin/bash -# Open a connection as a user +# Run SQL instructions in a database ($db_name by default) # -# usage: ynh_mysql_execute --user=user --password=password [--database=database] -# | arg: -u, --user= - the user name to connect as (by default, $db_user) -# | arg: -p, --password= - the user password (by default, $db_pwd) -# | arg: -d, --database= - the database to connect to (by default, $db_name) +# usage: ynh_mysql_db_shell [database] <<< "instructions" +# | arg: database= - the database to connect to (by default, $db_name) # # examples: -# ynh_mysql_execute <<< "UPDATE ...;" -# ynh_mysql_execute < /path/to/file.sql +# ynh_mysql_db_shell $db_name <<< "UPDATE ...;" +# ynh_mysql_db_shell < /path/to/file.sql # -# Requires YunoHost version 2.2.4 or higher. -ynh_mysql_execute() { - # ============ Argument parsing ============= - local -A args_array=([u]=user= [p]=password= [d]=database=) - local user - local password - local database - ynh_handle_getopts_args "$@" - user="${database:-$db_name}" - password="${database:-$db_pwd}" - database="${database:-$db_name}" - # =========================================== - - mysql --user="$user" --password="$password" --batch "$database" -} - -# Execute a command as root user -# -# usage: ynh_mysql_execute_as_root --sql=sql [--database=database] -# | arg: -s, --sql= - the SQL command to execute -# | arg: -d, --database= - the database to connect to (by default, $db_name) -# -# Requires YunoHost version 2.2.4 or higher. -ynh_mysql_execute_as_root() { - # ============ Argument parsing ============= - local -A args_array=([s]=sql= [d]=database=) - local sql - local database - ynh_handle_getopts_args "$@" - database="${database:-$db_name}" - # =========================================== - - if [ -n "$database" ]; then - database="--database=$database" - fi - - mysql -B "$database" <<<"$sql" -} - -# Execute a command from a file as root user -# -# usage: ynh_mysql_execute_file_as_root --file=file [--database=database] -# | arg: -f, --file= - the file containing SQL commands -# | arg: -d, --database= - the database to connect to (by default, $db_name) -# -# Requires YunoHost version 2.2.4 or higher. -ynh_mysql_execute_file_as_root() { - # ============ Argument parsing ============= - local -A args_array=([f]=file= [d]=database=) - local file - local database - ynh_handle_getopts_args "$@" - database="${database:-$db_name}" - # =========================================== - - if [ -n "$database" ]; then - database="--database=$database" - fi - - mysql -B "$database" <"$file" +ynh_mysql_db_shell() { + local database=${1:-$db_name} + mysql -B $database } # Create a database and grant optionnaly privilegies to a user @@ -82,7 +23,6 @@ ynh_mysql_execute_file_as_root() { # | arg: user - the user to grant privilegies # | arg: pwd - the password to identify user by # -# Requires YunoHost version 2.2.4 or higher. ynh_mysql_create_db() { local db=$1 @@ -97,7 +37,7 @@ ynh_mysql_create_db() { sql+=" WITH GRANT OPTION;" fi - ynh_mysql_execute_as_root --sql="$sql" + mysql -B <<< "$sql" } # Drop a database @@ -110,28 +50,20 @@ ynh_mysql_create_db() { # usage: ynh_mysql_drop_db db # | arg: db - the database name to drop # -# Requires YunoHost version 2.2.4 or higher. ynh_mysql_drop_db() { - ynh_mysql_execute_as_root --sql="DROP DATABASE ${1};" + mysql -B <<< "DROP DATABASE ${1};" } # Dump a database # -# usage: ynh_mysql_dump_db --database=database +# usage: ynh_mysql_dump_db database # | arg: -d, --database= - the database name to dump (by default, $db_name) # | ret: The mysqldump output # -# example: ynh_mysql_dump_db --database=roundcube > ./dump.sql +# example: ynh_mysql_dump_db "roundcube" > ./dump.sql # -# Requires YunoHost version 2.2.4 or higher. ynh_mysql_dump_db() { - # ============ Argument parsing ============= - local -A args_array=([d]=database=) - local database - ynh_handle_getopts_args "$@" - database="${database:-$db_name}" - # =========================================== - + local database=${1:-$db_name} mysqldump --single-transaction --skip-dump-date --routines "$database" } @@ -143,33 +75,31 @@ ynh_mysql_dump_db() { # | arg: user - the user name to create # | arg: pwd - the password to identify user by # -# Requires YunoHost version 2.2.4 or higher. ynh_mysql_create_user() { - ynh_mysql_execute_as_root \ - --sql="CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';" + mysql -B <<< "CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';" } # Check if a mysql user exists # # [internal] # -# usage: ynh_mysql_user_exists --user=user -# | arg: -u, --user= - the user for which to check existence +# usage: ynh_mysql_user_exists user +# | arg: user= - the user for which to check existence # | ret: 0 if the user exists, 1 otherwise. -# -# Requires YunoHost version 2.2.4 or higher. ynh_mysql_user_exists() { - # ============ Argument parsing ============= - local -A args_array=([u]=user=) - local user - ynh_handle_getopts_args "$@" - # =========================================== + local user=$1 + [[ -n "$(mysql -B <<< "SELECT User from mysql.user WHERE User = '$user';")" ]] +} - if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]; then - return 1 - else - return 0 - fi +# Check if a mysql database exists +# +# usage: ynh_mysql_database_exists database +# | arg: database - the database for which to check existence +# | exit: Return 1 if the database doesn't exist, 0 otherwise +# +ynh_mysql_database_exists() { + local database=$1 + mysqlshow | grep -q "^| $database " } # Drop a user @@ -179,70 +109,6 @@ ynh_mysql_user_exists() { # usage: ynh_mysql_drop_user user # | arg: user - the user name to drop # -# Requires YunoHost version 2.2.4 or higher. ynh_mysql_drop_user() { - ynh_mysql_execute_as_root --sql="DROP USER '${1}'@'localhost';" -} - -# Create a database, an user and its password. Then store the password in the app's config -# -# [packagingv1] -# -# usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database -# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated -# -# After executing this helper, the password of the created database will be available in `$db_pwd` -# It will also be stored as "`mysqlpwd`" into the app settings. -# -# Requires YunoHost version 2.6.4 or higher. -ynh_mysql_setup_db() { - # ============ Argument parsing ============= - local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) - local db_user - local db_name - db_pwd="" - ynh_handle_getopts_args "$@" - # =========================================== - - # Generate a random password - local new_db_pwd=$(ynh_string_random) - # If $db_pwd is not provided, use new_db_pwd instead for db_pwd - db_pwd="${db_pwd:-$new_db_pwd}" - - # Dirty patch for super-legacy apps - dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn --message="Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } - - ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" - ynh_app_setting_set --key=mysqlpwd --value=$db_pwd -} - -# Remove a database if it exists, and the associated user -# -# [packagingv1] -# -# usage: ynh_mysql_remove_db --db_user=user --db_name=name -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database -# -# Requires YunoHost version 2.6.4 or higher. -ynh_mysql_remove_db() { - # ============ Argument parsing ============= - local -Ar args_array=([u]=db_user= [n]=db_name=) - local db_user - local db_name - ynh_handle_getopts_args "$@" - # =========================================== - - if mysqlshow | grep -q "^| $db_name "; then - ynh_mysql_drop_db $db_name - else - ynh_print_warn --message="Database $db_name not found" - fi - - # Remove mysql user if it exists - if ynh_mysql_user_exists --user=$db_user; then - ynh_mysql_drop_user $db_user - fi + mysql -B <<< "DROP USER '${1}'@'localhost';" } diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 4e2a3bb24..713743ec0 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -3,79 +3,18 @@ PSQL_ROOT_PWD_FILE=/etc/yunohost/psql PSQL_VERSION=13 -# Open a connection as a user +# Run SQL instructions in a database ($db_name by default) # -# usage: ynh_psql_execute --user=user --password=password [--database=database] -# | arg: -u, --user= - the user name to connect as (by default, $db_user) -# | arg: -p, --password= - the user password (by default, $db_pw) -# | arg: -d, --database= - the database to connect to (by default, $db_name) +# usage: ynh_psql_db_shell database <<< "instructions" +# | arg: database - the database to connect to (by default, $db_name) # # examples: -# ynh_psql_execute <<< "UPDATE ...;" -# ynh_psql_execute < /path/to/file.sql +# ynh_psql_db_shell $db_name <<< "UPDATE ...;" +# ynh_psql_db_shell < /path/to/file.sql # -# Requires YunoHost version 3.5.0 or higher. -ynh_psql_execute() { - # ============ Argument parsing ============= - local -A args_array=([u]=user= [p]=password= [d]=database=) - local user - local password - local database - ynh_handle_getopts_args "$@" - user="${user:-$db_user}" - password="${password:-$db_pwd}" - database="${database:-$db_name}" - # =========================================== - - sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" -} - -# Execute a command as root user -# -# usage: ynh_psql_execute_as_root --sql=sql [--database=database] -# | arg: -s, --sql= - the SQL command to execute -# | arg: -d, --database= - the database to connect to (by default, $db_name) -# -# Requires YunoHost version 3.5.0 or higher. -ynh_psql_execute_as_root() { - # ============ Argument parsing ============= - local -A args_array=([s]=sql= [d]=database=) - local sql - local database - ynh_handle_getopts_args "$@" - database="${database:-$db_name}" - # =========================================== - - if [ -n "$database" ]; then - database="--database=$database" - fi - - ynh_psql_execute --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ - --database="$database" <<<"$sql" -} - -# Execute a command from a file as root user -# -# usage: ynh_psql_execute_file_as_root --file=file [--database=database] -# | arg: -f, --file= - the file containing SQL commands -# | arg: -d, --database= - the database to connect to (by default, $db_name) -# -# Requires YunoHost version 3.5.0 or higher. -ynh_psql_execute_file_as_root() { - # ============ Argument parsing ============= - local -A args_array=([f]=file= [d]=database=) - local file - local database - ynh_handle_getopts_args "$@" - database="${database:-$db_name}" - # =========================================== - - if [ -n "$database" ]; then - database="--database=$database" - fi - - ynh_psql_execute --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ - --database="$database" <"$file" +ynh_psql_db_shell() { + local database="${1:-$db_name}" + sudo --login --user=postgres psql "$database" } # Create a database and grant optionnaly privilegies to a user @@ -86,7 +25,6 @@ ynh_psql_execute_file_as_root() { # | arg: db - the database name to create # | arg: user - the user to grant privilegies # -# Requires YunoHost version 3.5.0 or higher. ynh_psql_create_db() { local db=$1 local user=${2:-} @@ -99,7 +37,7 @@ ynh_psql_create_db() { sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi - ynh_psql_execute_as_root --sql="$sql" + sudo --login --user=postgres psql <<< "$sql" } # Drop a database @@ -112,33 +50,25 @@ ynh_psql_create_db() { # usage: ynh_psql_drop_db db # | arg: db - the database name to drop # -# Requires YunoHost version 3.5.0 or higher. ynh_psql_drop_db() { local db=$1 # First, force disconnection of all clients connected to the database # https://stackoverflow.com/questions/17449420/postgresql-unable-to-drop-database-because-of-some-auto-connections-to-db - ynh_psql_execute_as_root --sql="REVOKE CONNECT ON DATABASE $db FROM public;" --database="$db" - ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db' AND pid <> pg_backend_pid();" --database="$db" + sudo --login --user=postgres psql $db <<< "REVOKE CONNECT ON DATABASE $db FROM public;" + sudo --login --user=postgres psql $db <<< "SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db' AND pid <> pg_backend_pid();" sudo --login --user=postgres dropdb $db } # Dump a database # -# usage: ynh_psql_dump_db --database=database -# | arg: -d, --database= - the database name to dump (by default, $db_name) +# usage: ynh_psql_dump_db database +# | arg: database - the database name to dump (by default, $db_name) # | ret: the psqldump output # # example: ynh_psql_dump_db 'roundcube' > ./dump.sql # -# Requires YunoHost version 3.5.0 or higher. ynh_psql_dump_db() { - # ============ Argument parsing ============= - local -A args_array=([d]=database=) - local database - ynh_handle_getopts_args "$@" - database="${database:-$db_name}" - # =========================================== - + local database="${1:-$db_name}" sudo --login --user=postgres pg_dump "$database" } @@ -150,62 +80,32 @@ ynh_psql_dump_db() { # | arg: user - the user name to create # | arg: pwd - the password to identify user by # -# Requires YunoHost version 3.5.0 or higher. ynh_psql_create_user() { - local user=$1 - local pwd=$2 - ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" + sudo --login --user=postgres psql <<< "CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" } # Check if a psql user exists # # [packagingv1] # -# usage: ynh_psql_user_exists --user=user -# | arg: -u, --user= - the user for which to check existence +# usage: ynh_psql_user_exists user +# | arg: user= - the user for which to check existence # | exit: Return 1 if the user doesn't exist, 0 otherwise # -# Requires YunoHost version 3.5.0 or higher. ynh_psql_user_exists() { - # ============ Argument parsing ============= - local -A args_array=([u]=user=) - local user - ynh_handle_getopts_args "$@" - # =========================================== - - if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user"; then - return 1 - else - return 0 - fi + local user=$1 + sudo --login --user=postgres psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user" } # Check if a psql database exists # -# usage: ynh_psql_database_exists --database=database -# | arg: -d, --database= - the database for which to check existence (by default, $db_name) +# usage: ynh_psql_database_exists database +# | arg: database - the database for which to check existence # | exit: Return 1 if the database doesn't exist, 0 otherwise # -# Requires YunoHost version 3.5.0 or higher. ynh_psql_database_exists() { - # ============ Argument parsing ============= - local -A args_array=([d]=database=) - local database - ynh_handle_getopts_args "$@" - database="${database:-$db_name}" - # =========================================== - - # if psql is not there, we cannot check the db - # though it could exists. - if ! command -v psql - then - ynh_print_warn --message="PostgreSQL is not installed, impossible to check for db existence." - return 1 - elif ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then - return 1 - else - return 0 - fi + local database=$1 + sudo --login --user=postgres psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database" } # Drop a user @@ -215,72 +115,6 @@ ynh_psql_database_exists() { # usage: ynh_psql_drop_user user # | arg: user - the user name to drop # -# Requires YunoHost version 3.5.0 or higher. ynh_psql_drop_user() { - ynh_psql_execute_as_root --sql="DROP USER ${1};" -} - -# Create a database, an user and its password. Then store the password in the app's config -# -# [packagingv1] -# -# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database -# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated -# -# After executing this helper, the password of the created database will be available in $db_pwd -# -# Requires YunoHost version 2.7.13 or higher. -ynh_psql_setup_db() { - # ============ Argument parsing ============= - local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) - local db_user - local db_name - db_pwd="" - ynh_handle_getopts_args "$@" - # =========================================== - - if ! ynh_psql_user_exists --user=$db_user; then - local new_db_pwd=$(ynh_string_random) # Generate a random password - # If $db_pwd is not provided, use new_db_pwd instead for db_pwd - db_pwd="${db_pwd:-$new_db_pwd}" - - ynh_psql_create_user "$db_user" "$db_pwd" - elif [ -z $db_pwd ]; then - ynh_die --message="The user $db_user exists, please provide his password" - fi - - ynh_psql_create_db "$db_name" "$db_user" # Create the database -} - -# Remove a database if it exists, and the associated user -# -# [packagingv1] -# -# usage: ynh_psql_remove_db --db_user=user --db_name=name -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database -# -# Requires YunoHost version 2.7.13 or higher. -ynh_psql_remove_db() { - # ============ Argument parsing ============= - local -A args_array=([u]=db_user= [n]=db_name=) - local db_user - local db_name - ynh_handle_getopts_args "$@" - # =========================================== - - if ynh_psql_database_exists --database=$db_name; then # Check if the database exists - ynh_psql_drop_db $db_name # Remove the database - else - ynh_print_warn --message="Database $db_name not found" - fi - - # Remove psql user if it exists - if ynh_psql_user_exists --user=$db_user; then - ynh_psql_drop_user $db_user - else - ynh_print_warn --message="User $db_user not found" - fi + sudo --login --user=postgres psql <<< "DROP USER ${1};" } diff --git a/src/utils/resources.py b/src/utils/resources.py index e2400e91d..21f74b191 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -1455,13 +1455,16 @@ class DatabaseAppResource(AppResource): db_name = self.get_setting("db_name") or db_user if self.dbtype == "mysql": - self._run_script( - "deprovision", f"ynh_mysql_remove_db '{db_name}' '{db_user}'" - ) + db_helper_name = "mysql" elif self.dbtype == "postgresql": - self._run_script( - "deprovision", f"ynh_psql_remove_db '{db_name}' '{db_user}'" - ) + db_helper_name = "psql" + + self._run_script( + "deprovision", f""" +ynh_{db_helper_name}_database_exists "{db_name}" && ynh_{db_helper_name}_drop_db "{db_name}" || true +ynh_{db_helper_name}_user_exists "{db_user}" && ynh_{db_helper_name}_drop_user "{db_user}" || true +""" + ) self.delete_setting("db_name") self.delete_setting("db_user") From 5a6a8e6c7312739e6c99117a78085df65eed8e47 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 16 Jun 2024 14:12:11 +0200 Subject: [PATCH 068/146] helpers2.1: fix unecessary change in ynh_read_manifest, key shouldnt need to be prefixed with . --- helpers/helpers.v2.1.d/utils | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 8326f644f..72196394b 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -148,7 +148,7 @@ ynh_setup_source() { source_id="${source_id:-main}" # =========================================== - local sources_json=$(ynh_read_manifest ".resources.sources[\"$source_id\"]") + local sources_json=$(ynh_read_manifest "resources.sources[\"$source_id\"]") if jq -re ".url" <<< "$sources_json" then local arch_prefix="" @@ -456,7 +456,7 @@ ynh_safe_rm() { # # Requires YunoHost version 3.5.0 or higher. ynh_read_manifest() { - cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq "$1" --raw-output + cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".$1" --raw-output } # Return the app upstream version, deduced from `$YNH_APP_MANIFEST_VERSION` and strippig the `~ynhX` part From bd43a4504e715a84cc769ce17caed1356a0d59c3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin <4533074+alexAubin@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:12:11 +0200 Subject: [PATCH 069/146] Update main.cf: fuck postfix syntax --- conf/postfix/main.cf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/postfix/main.cf b/conf/postfix/main.cf index 957907dd3..01af1b619 100644 --- a/conf/postfix/main.cf +++ b/conf/postfix/main.cf @@ -111,8 +111,7 @@ smtpd_sender_login_maps = unionmap:{ # Regular Yunohost accounts ldap:/etc/postfix/ldap-accounts.cf, # Extra maps for app system users who need to send emails - hash:/etc/postfix/app_senders_login_maps -} + hash:/etc/postfix/app_senders_login_maps } # Dovecot LDA virtual_transport = dovecot From 8f8070983d78b26fa387f21ee55d282b9cb77e6f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2024 04:07:07 +0200 Subject: [PATCH 070/146] helpers2.1: rework the 'apt' helper: effectivement call them ynh_install/remove_apt_dependencies (instead of 'app_dependencies'...), remove unused stuff, bloat and unecessary non-linear flows... --- helpers/helpers.v2.1.d/apt | 446 ++++++++++++--------------------- helpers/helpers.v2.1.d/mongodb | 2 +- 2 files changed, 155 insertions(+), 293 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 38eb03c13..03a645611 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -1,219 +1,6 @@ #!/bin/bash -# Check if apt is free to use, or wait, until timeout. -# -# [internal] -# -# usage: ynh_wait_dpkg_free -# | exit: Return 1 if dpkg is broken -# -# Requires YunoHost version 3.3.1 or higher. -ynh_wait_dpkg_free() { - local try - set +o xtrace # set +x - # With seq 1 17, timeout will be almost 30 minutes - for try in $(seq 1 17); do - # Check if /var/lib/dpkg/lock is used by another process - if lsof /var/lib/dpkg/lock >/dev/null; then - echo "apt is already in use..." - # Sleep an exponential time at each round - sleep $((try * try)) - else - # Check if dpkg hasn't been interrupted and is fully available. - # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 - local dpkg_dir="/var/lib/dpkg/updates/" - - # For each file in $dpkg_dir - while read dpkg_file <&9; do - # Check if the name of this file contains only numbers. - if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then - # If so, that a remaining of dpkg. - ynh_print_warn --message="dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." - set -o xtrace # set -x - return 1 - fi - done 9<<<"$(ls -1 $dpkg_dir)" - set -o xtrace # set -x - return 0 - fi - done - echo "apt still used, but timeout reached !" - set -o xtrace # set -x -} - -# Check either a package is installed or not -# -# example: ynh_package_is_installed foobar && echo "installed" -# -# usage: ynh_package_is_installed name -# | arg: name - the package name to check -# | ret: 0 if the package is installed, 1 else. -# -# Requires YunoHost version 2.2.4 or higher. -ynh_package_is_installed() { - local package=$1 - dpkg-query --show --showformat='${Status}' "$package" 2>/dev/null \ - | grep --count "ok installed" &>/dev/null -} - -# Get the version of an installed package -# -# example: version=$(ynh_package_version --package=yunohost) -# -# [internal] -# -# usage: ynh_package_version --package=name -# | arg: -p, --package= - the package name to get version -# | ret: the version or an empty string -# -# Requires YunoHost version 2.2.4 or higher. -ynh_package_version() { - # ============ Argument parsing ============= - local -A args_array=([p]=package=) - local package - ynh_handle_getopts_args "$@" - # =========================================== - - if ynh_package_is_installed "$package"; then - dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null - else - echo '' - fi -} - -# APT wrapper for non-interactive operation -# -# [internal] -# -# usage: ynh_apt update -# -# Requires YunoHost version 2.4.0.3 or higher. -ynh_apt() { - ynh_wait_dpkg_free - LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0 $@ -} - -# Update package index files -# -# [internal] -# -# usage: ynh_package_update -# -# Requires YunoHost version 2.2.4 or higher. -ynh_package_update() { - ynh_apt update -} - -# Install package(s) -# -# [internal] -# -# usage: ynh_package_install name [name [...]] -# | arg: name - the package name to install -# -# Requires YunoHost version 2.2.4 or higher. -ynh_package_install() { - ynh_apt --no-remove --option Dpkg::Options::=--force-confdef \ - --option Dpkg::Options::=--force-confold install $@ -} - -# Remove package(s) -# -# [internal] -# -# usage: ynh_package_remove name [name [...]] -# | arg: name - the package name to remove -# -# Requires YunoHost version 2.2.4 or higher. -ynh_package_remove() { - ynh_apt remove $@ -} - -# Remove package(s) and their uneeded dependencies -# -# [internal] -# -# usage: ynh_package_autoremove name [name [...]] -# | arg: name - the package name to remove -# -# Requires YunoHost version 2.2.4 or higher. -ynh_package_autoremove() { - ynh_apt autoremove $@ -} - -# Purge package(s) and their uneeded dependencies -# -# [internal] -# -# usage: ynh_package_autopurge name [name [...]] -# | arg: name - the package name to autoremove and purge -# -# Requires YunoHost version 2.7.2 or higher. -ynh_package_autopurge() { - ynh_apt autoremove --purge $@ -} - -# Build and install a package from an equivs control file -# -# [internal] -# -# example: generate an empty control file with `equivs-control`, adjust its -# content and use helper to build and install the package: -# ynh_package_install_from_equivs /path/to/controlfile -# -# usage: ynh_package_install_from_equivs controlfile -# | arg: controlfile - path of the equivs control file -# -# Requires YunoHost version 2.2.4 or higher. -ynh_package_install_from_equivs() { - local controlfile=$1 - - # retrieve package information - local pkgname=$(grep '^Package: ' $controlfile | cut --delimiter=' ' --fields=2) # Retrieve the name of the debian package - local pkgversion=$(grep '^Version: ' $controlfile | cut --delimiter=' ' --fields=2) # And its version number - [[ -z "$pkgname" || -z "$pkgversion" ]] \ - && ynh_die --message="Invalid control file" # Check if this 2 variables aren't empty. - - # Update packages cache - ynh_package_update - - # Build and install the package - local TMPDIR=$(mktemp --directory) - - # Make sure to delete the legacy compat file - # It's now handle somewhat magically through the control file - rm -f /usr/share/equivs/template/debian/compat - - # Note that the cd executes into a sub shell - # Create a fake deb package with equivs-build and the given control file - # Install the fake package without its dependencies with dpkg - # Install missing dependencies with ynh_package_install - ynh_wait_dpkg_free - cp "$controlfile" "${TMPDIR}/control" - ( - cd "$TMPDIR" - LC_ALL=C equivs-build ./control 2>&1 - LC_ALL=C dpkg --force-depends --install "./${pkgname}_${pkgversion}_all.deb" 2>&1 | tee ./dpkg_log - ) - - ynh_package_install --fix-broken \ - || { # If the installation failed - # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) - # Parse the list of problematic dependencies from dpkg's log ... - # (relevant lines look like: "foo-ynh-deps depends on bar; however:") - local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')" - # Fake an install of those dependencies to see the errors - # The sed command here is, Print only from 'Reading state info' to the end. - [[ -n "$problematic_dependencies" ]] && ynh_package_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 - ynh_die --message="Unable to install dependencies" - } - [[ -n "$TMPDIR" ]] && rm --recursive --force $TMPDIR # Remove the temp dir. - - # check if the package is actually installed - ynh_package_is_installed "$pkgname" -} - -YNH_INSTALL_APP_DEPENDENCIES_REPLACE="true" +YNH_INSTALL_APT_DEPENDENCIES_REPLACE="true" # Define and install dependencies with a equivs control file # @@ -228,17 +15,13 @@ YNH_INSTALL_APP_DEPENDENCIES_REPLACE="true" # | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc). # # Requires YunoHost version 2.6.4 or higher. -ynh_install_app_dependencies() { +ynh_install_apt_dependencies() { local dependencies=$@ # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" local dependencies=${dependencies//|/ | } - local version=$(ynh_read_manifest "version") - if [ -z "${version}" ] || [ "$version" == "null" ]; then - version="1.0" - fi - local dep_app=${app//_/-} # Replace all '_' by '-' + local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps # Handle specific versions if [[ "$dependencies" =~ [\<=\>] ]]; then @@ -253,6 +36,10 @@ ynh_install_app_dependencies() { dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')" fi + # ############################## # + # Specific tweaks related to PHP # + # ############################## # + # Check for specific php dependencies which requires sury # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni" # The (?<=php) syntax corresponds to lookbehind ;) @@ -291,30 +78,37 @@ ynh_install_app_dependencies() { ynh_app_setting_set --key=php_version --value=$YNH_DEFAULT_PHP_VERSION fi - local psql_installed="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)" + # Specific tweak related to Postgresql (cf end of the helper) + local psql_installed="$(_ynh_apt_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)" # The first time we run ynh_install_app_dependencies, we will replace the # entire control file (This is in particular meant to cover the case of # upgrade script where ynh_install_app_dependencies is called with this # expected effect) Otherwise, any subsequent call will add dependencies # to those already present in the equivs control file. - if [[ $YNH_INSTALL_APP_DEPENDENCIES_REPLACE == "true" ]] + if [[ $YNH_INSTALL_APT_DEPENDENCIES_REPLACE == "true" ]] then - YNH_INSTALL_APP_DEPENDENCIES_REPLACE="false" + YNH_INSTALL_APT_DEPENDENCIES_REPLACE="false" else local current_dependencies="" - if ynh_package_is_installed "${dep_app}-ynh-deps" + if _ynh_apt_package_is_installed "${app_ynh_deps}" then - current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) " + current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${app_ynh_deps}) " current_dependencies=${current_dependencies// | /|} fi dependencies="$current_dependencies, $dependencies" fi - cat >/tmp/${dep_app}-ynh-deps.control <${TMPDIR}/control <&1 + LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" 2>&1 | tee ./dpkg_log + ) + + # Then install the missing dependencies with apt install + _ynh_apt_install --fix-broken \ + || { # If the installation failed + # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) + # Parse the list of problematic dependencies from dpkg's log ... + # (relevant lines look like: "foo-ynh-deps depends on bar; however:") + local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')" + # Fake an install of those dependencies to see the errors + # The sed command here is, Print only from 'Reading state info' to the end. + [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 + ynh_die --message="Unable to install dependencies" + } + rm --recursive --force "$TMPDIR" # Remove the temp dir. + + # check if the package is actually installed + _ynh_apt_package_is_installed "${app_ynh_deps}" || ynh_die --message="Unable to install dependencies" + + # Specific tweak related to Postgresql + # -> trigger postgresql regenconf if we may have just installed postgresql + local psql_installed2="$(_ynh_apt_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)" if [[ "$psql_installed" != "$psql_installed2" ]] then yunohost tools regen-conf postgresql @@ -341,31 +165,31 @@ EOF # # Dependencies will removed only if no other package need them. # -# usage: ynh_remove_app_dependencies +# usage: ynh_remove_apt_dependencies # # Requires YunoHost version 2.6.4 or higher. -ynh_remove_app_dependencies() { - local dep_app=${app//_/-} # Replace all '_' by '-' +ynh_remove_apt_dependencies() { + local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps local current_dependencies="" - if ynh_package_is_installed "${dep_app}-ynh-deps"; then - current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) " + if _ynh_apt_package_is_installed "${app_ynh_deps}"; then + current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${app_ynh_deps}) " current_dependencies=${current_dependencies// | /|} fi # Edge case where the app dep may be on hold, # cf https://forum.yunohost.org/t/migration-error-cause-of-ffsync/20675/4 - if apt-mark showhold | grep -q -w ${dep_app}-ynh-deps + if apt-mark showhold | grep -q -w ${app_ynh_deps} then - apt-mark unhold ${dep_app}-ynh-deps + apt-mark unhold ${app_ynh_deps} fi # Remove the fake package and its dependencies if they not still used. # (except if dpkg doesn't know anything about the package, # which should be symptomatic of a failed install, and we don't want bash to report an error) - if dpkg-query --show ${dep_app}-ynh-deps &>/dev/null + if dpkg-query --show ${app_ynh_deps} &>/dev/null then - ynh_package_autopurge ${dep_app}-ynh-deps + _ynh_apt autoremove --purge ${app_ynh_deps} fi } @@ -373,13 +197,13 @@ ynh_remove_app_dependencies() { # # [packagingv1] # -# usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" --key=key_url +# usage: ynh_install_extra_apt_dependencies --repo="repo" --package="dep1 dep2" --key=key_url # | arg: -r, --repo= - Complete url of the extra repository. # | arg: -p, --package= - The packages to install from this extra repository # | arg: -k, --key= - url to get the public key. # # Requires YunoHost version 3.8.1 or higher. -ynh_install_extra_app_dependencies() { +ynh_install_extra_apt_dependencies() { # ============ Argument parsing ============= local -A args_array=([r]=repo= [p]=package= [k]=key=) local repo @@ -388,39 +212,6 @@ ynh_install_extra_app_dependencies() { ynh_handle_getopts_args "$@" # =========================================== - # Add an extra repository for those packages - ynh_install_extra_repo --repo="$repo" --key=$key - - # Install requested dependencies from this extra repository. - ynh_install_app_dependencies "$package" - - # Force to upgrade to the last version... - # Without doing apt install, an already installed dep is not upgraded - local apps_auto_installed="$(apt-mark showauto $package)" - ynh_package_install "$package" - [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed - - # Remove this extra repository after packages are installed - ynh_remove_extra_repo -} - -# Add an extra repository correctly, pin it and get the key. -# -# [internal] -# -# usage: ynh_install_extra_repo --repo="repo" [--key=key_url] -# | arg: -r, --repo= - Complete url of the extra repository. -# | arg: -k, --key= - url to get the public key. -# -# Requires YunoHost version 3.8.1 or higher. -ynh_install_extra_repo() { - # ============ Argument parsing ============= - local -A args_array=([r]=repo= [k]=key=) - local repo - local key - ynh_handle_getopts_args "$@" - # =========================================== - # Split the repository into uri, suite and components. repo="${repo#deb }" local uri="$(echo "$repo" | awk '{ print $1 }')" @@ -444,28 +235,99 @@ Pin: origin $pin Pin-Priority: 995 EOF - # Get the public key for the repo mkdir --parents "/etc/apt/trusted.gpg.d" # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | tee /etc/apt/trusted.gpg.d/$name.gpg >/dev/null - # Update the list of package with the new repo - ynh_package_update -} + # Update the list of package with the new repo NB: we use -o + # Dir::Etc::sourcelist to only refresh this repo, because + # ynh_install_apt_dependencies will also call an ynh_apt update on its own + # and it's good to limit unecessary requests ... Here we mainly want to + # validate that the url+key is correct before going further + _ynh_apt update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/$app.list" -# Remove an extra repository and the assiociated configuration. -# -# [internal] -# -# usage: ynh_remove_extra_repo -# -# Requires YunoHost version 3.8.1 or higher. -ynh_remove_extra_repo() { + # Install requested dependencies from this extra repository. + # NB: because of the mechanism with $YNH_INSTALL_APT_DEPENDENCIES_REPLACE, + # this will usually only *append* to the existing list of dependency, not + # replace the existing $app-ynh-deps + ynh_install_apt_dependencies "$package" + # Force to upgrade to the last version... + # Without doing apt install, an already installed dep is not upgraded + local apps_auto_installed="$(apt-mark showauto $package)" + _ynh_apt_install "$package" + [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed + + # Remove this extra repository after packages are installed ynh_safe_rm "/etc/apt/sources.list.d/$app.list" ynh_safe_rm "/etc/apt/preferences.d/$app" - if [ -e /etc/apt/trusted.gpg.d/$app.gpg ]; then - ynh_safe_rm "/etc/apt/trusted.gpg.d/$app.gpg" - fi - ynh_package_update + ynh_safe_rm "/etc/apt/trusted.gpg.d/$app.gpg" + ynh_apt_update +} + + +####################### +# Internal misc utils # +####################### + +# Check if apt is free to use, or wait, until timeout. +_ynh_wait_dpkg_free() { + local try + set +o xtrace # set +x + # With seq 1 17, timeout will be almost 30 minutes + for try in $(seq 1 17); do + # Check if /var/lib/dpkg/lock is used by another process + if lsof /var/lib/dpkg/lock >/dev/null; then + echo "apt is already in use..." + # Sleep an exponential time at each round + sleep $((try * try)) + else + # Check if dpkg hasn't been interrupted and is fully available. + # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + local dpkg_dir="/var/lib/dpkg/updates/" + + # For each file in $dpkg_dir + while read dpkg_file <&9; do + # Check if the name of this file contains only numbers. + if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then + # If so, that a remaining of dpkg. + ynh_print_warn --message="dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + set -o xtrace # set -x + return 1 + fi + done 9<<<"$(ls -1 $dpkg_dir)" + set -o xtrace # set -x + return 0 + fi + done + echo "apt still used, but timeout reached !" + set -o xtrace # set -x +} + +# Check either a package is installed or not +_ynh_apt_package_is_installed() { + local package=$1 + dpkg-query --show --showformat='${db:Status-Status}' "$package" 2>/dev/null \ + | grep --quiet "^installed$" &>/dev/null +} + +# Return the installed version of an apt package, if installed +_ynh_apt_package_version() { + if _ynh_apt_package_is_installed "$package"; then + dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null + else + echo '' + fi +} + +# APT wrapper for non-interactive operation +_ynh_apt() { + _ynh_wait_dpkg_free + LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0 $@ +} + +# Wrapper around "apt install" with the appropriate options +_ynh_apt_install() { + _ynh_apt --no-remove --option Dpkg::Options::=--force-confdef \ + --option Dpkg::Options::=--force-confold install $@ } diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index c6750855d..0eb9f4f8a 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -259,7 +259,7 @@ ynh_install_mongo() { # ynh_remove_mongo() { # Only remove the mongodb service if it is not installed. - if ! ynh_package_is_installed "mongodb*" + if ! _ynh_apt_package_is_installed "mongodb*" then ynh_print_info --message="Removing MongoDB service..." mongodb_servicename=mongod From ec354d443ddc4601e473a1613d31f7f327f0f9d1 Mon Sep 17 00:00:00 2001 From: Salamandar <6552989+Salamandar@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:53:55 +0200 Subject: [PATCH 071/146] helpers-2.1: Fix mysqlshow regex to list existing databases Co-authored-by: Alexandre Aubin <4533074+alexAubin@users.noreply.github.com> --- helpers/helpers.v1.d/mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v1.d/mysql b/helpers/helpers.v1.d/mysql index 98e9a745e..93479ce3c 100644 --- a/helpers/helpers.v1.d/mysql +++ b/helpers/helpers.v1.d/mysql @@ -184,7 +184,7 @@ ynh_mysql_user_exists() { # ynh_mysql_database_exists() { local database=$1 - mysqlshow | grep -q "^| $database " + mysqlshow | grep -qE "^|\s+$database\s+|" } # Drop a user From 218bf107fbb674caa8b34fa579695ac90f7899db Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2024 17:05:50 +0200 Subject: [PATCH 072/146] helpers2.1: rename everything again, i.e. ynh_{nodejs|ruby|composer|...}_install/remove (to have a proper ynh_{tech}_ prefix like we have for mysql/psql helpers) + same idea for ynh_config_add_{nginx|phpfpm|systemd|...} instead of ynh_add_foo_config --- helpers/helpers.v2.1.d/apt | 22 +++++++++++----------- helpers/helpers.v2.1.d/backup | 9 --------- helpers/helpers.v2.1.d/fail2ban | 18 +++++++++--------- helpers/helpers.v2.1.d/go | 18 +++++++++--------- helpers/helpers.v2.1.d/logrotate | 6 +++--- helpers/helpers.v2.1.d/nginx | 22 +++++++++++----------- helpers/helpers.v2.1.d/nodejs | 18 +++++++++--------- helpers/helpers.v2.1.d/php | 30 +++++++++++++++--------------- helpers/helpers.v2.1.d/ruby | 20 +++++++++----------- helpers/helpers.v2.1.d/systemd | 23 +++++++++-------------- helpers/helpers.v2.1.d/templating | 10 +++++----- helpers/helpers.v2.1.d/utils | 2 +- 12 files changed, 91 insertions(+), 107 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 03a645611..2cc1e33d1 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -1,6 +1,6 @@ #!/bin/bash -YNH_INSTALL_APT_DEPENDENCIES_REPLACE="true" +YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true" # Define and install dependencies with a equivs control file # @@ -15,7 +15,7 @@ YNH_INSTALL_APT_DEPENDENCIES_REPLACE="true" # | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc). # # Requires YunoHost version 2.6.4 or higher. -ynh_install_apt_dependencies() { +ynh_apt_install_dependencies() { local dependencies=$@ # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" @@ -86,9 +86,9 @@ ynh_install_apt_dependencies() { # upgrade script where ynh_install_app_dependencies is called with this # expected effect) Otherwise, any subsequent call will add dependencies # to those already present in the equivs control file. - if [[ $YNH_INSTALL_APT_DEPENDENCIES_REPLACE == "true" ]] + if [[ $YNH_APT_INSTALL_DEPENDENCIES_REPLACE == "true" ]] then - YNH_INSTALL_APT_DEPENDENCIES_REPLACE="false" + YNH_APT_INSTALL_DEPENDENCIES_REPLACE="false" else local current_dependencies="" if _ynh_apt_package_is_installed "${app_ynh_deps}" @@ -165,10 +165,10 @@ EOF # # Dependencies will removed only if no other package need them. # -# usage: ynh_remove_apt_dependencies +# usage: ynh_apt_remove_dependencies # # Requires YunoHost version 2.6.4 or higher. -ynh_remove_apt_dependencies() { +ynh_apt_remove_dependencies() { local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps local current_dependencies="" @@ -197,13 +197,13 @@ ynh_remove_apt_dependencies() { # # [packagingv1] # -# usage: ynh_install_extra_apt_dependencies --repo="repo" --package="dep1 dep2" --key=key_url +# usage: ynh_apt_install_dependencies_from_extra_repository --repo="repo" --package="dep1 dep2" --key=key_url # | arg: -r, --repo= - Complete url of the extra repository. # | arg: -p, --package= - The packages to install from this extra repository # | arg: -k, --key= - url to get the public key. # # Requires YunoHost version 3.8.1 or higher. -ynh_install_extra_apt_dependencies() { +ynh_apt_install_dependencies_from_extra_repository() { # ============ Argument parsing ============= local -A args_array=([r]=repo= [p]=package= [k]=key=) local repo @@ -241,16 +241,16 @@ EOF # Update the list of package with the new repo NB: we use -o # Dir::Etc::sourcelist to only refresh this repo, because - # ynh_install_apt_dependencies will also call an ynh_apt update on its own + # ynh_apt_install_dependencies will also call an ynh_apt update on its own # and it's good to limit unecessary requests ... Here we mainly want to # validate that the url+key is correct before going further _ynh_apt update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/$app.list" # Install requested dependencies from this extra repository. - # NB: because of the mechanism with $YNH_INSTALL_APT_DEPENDENCIES_REPLACE, + # NB: because of the mechanism with $ynh_apt_install_DEPENDENCIES_REPLACE, # this will usually only *append* to the existing list of dependency, not # replace the existing $app-ynh-deps - ynh_install_apt_dependencies "$package" + ynh_apt_install_dependencies "$package" # Force to upgrade to the last version... # Without doing apt install, an already installed dep is not upgraded diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 642c66e65..c0acbaa8d 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -307,12 +307,3 @@ ynh_delete_file_checksum() { local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' ynh_app_setting_delete --key=$checksum_setting_name } - -# Checks a backup archive exists -# -# [internal] -# -ynh_backup_archive_exists() { - yunohost backup list --output-as json --quiet \ - | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null -} diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index cfc768e13..de3636f3b 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -2,15 +2,15 @@ # Create a dedicated fail2ban config (jail and filter conf files) # -# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter +# usage 1: ynh_config_add_fail2ban --logpath=log_file --failregex=filter # | arg: -l, --logpath= - Log file to be checked by fail2ban # | arg: -r, --failregex= - Failregex to be looked for by fail2ban # -# usage 2: ynh_add_fail2ban_config +# usage 2: ynh_config_add_fail2ban # | arg: -t, --use_template - Use this helper in template mode # # This will use a template in `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf` -# See the documentation of `ynh_add_config` for a description of the template +# See the documentation of `ynh_config_add` for a description of the template # format and how placeholders are replaced with actual variables. # # Generally your template will look like that by example (for synapse): @@ -55,7 +55,7 @@ # ``` # # Requires YunoHost version 4.1.0 or higher. -ynh_add_fail2ban_config() { +ynh_config_add_fail2ban() { # ============ Argument parsing ============= local -A args_array=([l]=logpath= [r]=failregex=) local logpath @@ -65,7 +65,7 @@ ynh_add_fail2ban_config() { # If failregex is provided, Build a config file on-the-fly using $logpath and $failregex if [[ -n "${failregex:-}" ]]; then - test -n "$logpath" || ynh_die --message="ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$logpath" || ynh_die --message="ynh_config_add_fail2ban expects a logfile path as first argument and received nothing." echo " [__APP__] @@ -85,8 +85,8 @@ ignoreregex = " >"$YNH_APP_BASEDIR/conf/f2b_filter.conf" fi - ynh_add_config --template="f2b_jail.conf" --destination="/etc/fail2ban/jail.d/$app.conf" - ynh_add_config --template="f2b_filter.conf" --destination="/etc/fail2ban/filter.d/$app.conf" + ynh_config_add --template="f2b_jail.conf" --destination="/etc/fail2ban/jail.d/$app.conf" + ynh_config_add --template="f2b_filter.conf" --destination="/etc/fail2ban/filter.d/$app.conf" # if "$logpath" doesn't exist (as if using --use_template argument), assign # "$logpath" using the one in the previously generated fail2ban conf file @@ -116,10 +116,10 @@ ignoreregex = # Remove the dedicated fail2ban config (jail and filter conf files) # -# usage: ynh_remove_fail2ban_config +# usage: ynh_config_remove_fail2ban # # Requires YunoHost version 3.5.0 or higher. -ynh_remove_fail2ban_config() { +ynh_config_remove_fail2ban() { ynh_safe_rm "/etc/fail2ban/jail.d/$app.conf" ynh_safe_rm "/etc/fail2ban/filter.d/$app.conf" ynh_systemd_action --service=fail2ban --action=reload diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 12c089744..d28c396e9 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -48,12 +48,12 @@ _ynh_load_go_in_path_and_other_tweaks() { # When not possible (e.g. in systemd service definition), please use direct path # to goenv shims (e.g. $goenv_ROOT/shims/bundle) # -# usage: ynh_install_go +# usage: ynh_go_install # # Requires YunoHost version 3.2.2 or higher. -ynh_install_go () { +ynh_go_install () { - [[ -n "${go_version:-}" ]] || ynh_die --message="\$go_version should be defined prior to calling ynh_install_go" + [[ -n "${go_version:-}" ]] || ynh_die --message="\$go_version should be defined prior to calling ynh_go_install" # Load goenv path in PATH local CLEAR_PATH="$goenv_install_dir/bin:$PATH" @@ -118,7 +118,7 @@ ynh_install_go () { go_version=$final_go_version # Cleanup Go versions - ynh_cleanup_go + _ynh_go_cleanup # Set environment for Go users echo "#goenv @@ -137,8 +137,8 @@ eval \"\$(goenv init -)\" # # This helper will also cleanup Go versions # -# usage: ynh_remove_go -ynh_remove_go () { +# usage: ynh_go_remove +ynh_go_remove () { local go_version=$(ynh_app_setting_get --key="go_version") # Load goenv path in PATH @@ -151,7 +151,7 @@ ynh_remove_go () { ynh_app_setting_delete --key="go_version" # Cleanup Go versions - ynh_cleanup_go + _ynh_go_cleanup } # Remove no more needed versions of Go used by the app. @@ -160,8 +160,8 @@ ynh_remove_go () { # and uninstall them # If no app uses Go, goenv will be also removed. # -# usage: ynh_cleanup_go -ynh_cleanup_go () { +# usage: _ynh_go_cleanup +_ynh_go_cleanup () { # List required Go versions local installed_apps=$(yunohost app list --output-as json --quiet | jq -r .apps[].id) diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index c81e43d0c..5fce19d0b 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -4,7 +4,7 @@ FIRST_CALL_TO_LOGROTATE="true" # Add a logrotate configuration to manage log files / log directory # -# usage: ynh_add_logrotate_config [/path/to/log/file/or/folder] +# usage: ynh_config_add_logrotate [/path/to/log/file/or/folder] # # If not argument is provided, `/var/log/$app/*.log` is used as default. # @@ -12,7 +12,7 @@ FIRST_CALL_TO_LOGROTATE="true" # (ie it doesnt come from a specific app template like nginx or systemd conf) # # Requires YunoHost version 2.6.4 or higher. -ynh_add_logrotate_config() { +ynh_config_add_logrotate() { logfile="$1" @@ -70,7 +70,7 @@ EOF # usage: ynh_remove_logrotate # # Requires YunoHost version 2.6.4 or higher. -ynh_remove_logrotate_config() { +ynh_config_remove_logrotate() { if [ -e "/etc/logrotate.d/$app" ]; then rm "/etc/logrotate.d/$app" fi diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index 3a931fca8..b3b2836ae 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -2,13 +2,13 @@ # Create a dedicated nginx config # -# usage: ynh_add_nginx_config +# usage: ynh_config_add_nginx # # This will use a template in `../conf/nginx.conf` -# See the documentation of `ynh_add_config` for a description of the template +# See the documentation of `ynh_config_add` for a description of the template # format and how placeholders are replaced with actual variables. # -# Additionally, ynh_add_nginx_config will replace: +# Additionally, ynh_config_add_nginx will replace: # - `#sub_path_only` by empty string if `path` is not `'/'` # - `#root_path_only` by empty string if `path` *is* `'/'` # @@ -16,11 +16,11 @@ # location # # Requires YunoHost version 4.1.0 or higher. -ynh_add_nginx_config() { +ynh_config_add_nginx() { local finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf" - ynh_add_config --template="nginx.conf" --destination="$finalnginxconf" + ynh_config_add --template="nginx.conf" --destination="$finalnginxconf" if [ "${path:-}" != "/" ]; then ynh_replace --match="^#sub_path_only" --replace="" --file="$finalnginxconf" @@ -35,10 +35,10 @@ ynh_add_nginx_config() { # Remove the dedicated nginx config # -# usage: ynh_remove_nginx_config +# usage: ynh_config_remove_nginx # # Requires YunoHost version 2.7.2 or higher. -ynh_remove_nginx_config() { +ynh_config_remove_nginx() { ynh_safe_rm "/etc/nginx/conf.d/$domain.d/$app.conf" ynh_systemd_action --service=nginx --action=reload } @@ -46,14 +46,14 @@ ynh_remove_nginx_config() { # Regen the nginx config in a change url context # -# usage: ynh_change_url_nginx_config +# usage: ynh_config_change_url_nginx # # Requires YunoHost version 11.1.9 or higher. -ynh_change_url_nginx_config() { +ynh_config_change_url_nginx() { # Make a backup of the original NGINX config file if manually modified # (nb: this is possibly different from the same instruction called by - # ynh_add_config inside ynh_add_nginx_config because the path may have + # ynh_config_add inside ynh_config_add_nginx because the path may have # changed if we're changing the domain too...) local old_nginx_conf_path=/etc/nginx/conf.d/$old_domain.d/$app.conf ynh_backup_if_checksum_is_different --file="$old_nginx_conf_path" @@ -61,5 +61,5 @@ ynh_change_url_nginx_config() { ynh_safe_rm "$old_nginx_conf_path" # Regen the nginx conf - ynh_add_nginx_config + ynh_config_add_nginx } diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index 0b8788409..e47ab9eb0 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -30,7 +30,7 @@ _ynh_load_nodejs_in_path_and_other_tweaks() { # # The installed version is defined by $nodejs_version which should be defined as global prior to calling this helper # -# usage: ynh_install_nodejs +# usage: ynh_nodejs_install # # `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use. # That's how it changes the version @@ -41,10 +41,10 @@ _ynh_load_nodejs_in_path_and_other_tweaks() { # (Environment="PATH=__PATH_WITH_NODEJS__") # # Requires YunoHost version 2.7.12 or higher. -ynh_install_nodejs() { +ynh_nodejs_install() { # Use n, https://github.com/tj/n to manage the nodejs versions - [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_install_nodejs" + [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_nodejs_install" # Create $n_install_dir mkdir --parents "$n_install_dir" @@ -95,23 +95,23 @@ ynh_install_nodejs() { ynh_app_setting_set --key=nodejs_version --value=$nodejs_version # Build the update script and set the cronjob - ynh_cron_upgrade_node + _ynh_cron_upgrade_node _ynh_load_nodejs_in_path_and_other_tweaks } # Remove the version of node used by the app. # -# usage: ynh_remove_nodejs +# usage: ynh_nodejs_remove # # This helper will check if another app uses the same version of node. # - If not, this version of node will be removed. # - If no other app uses node, n will be also removed. # # Requires YunoHost version 2.7.12 or higher. -ynh_remove_nodejs() { +ynh_nodejs_remove() { - [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_install_nodejs" + [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_nodejs_remove" # Remove the line for this app sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version" @@ -136,10 +136,10 @@ ynh_remove_nodejs() { # # This cron will check and update all minor node versions used by your apps. # -# usage: ynh_cron_upgrade_node +# usage: _ynh_cron_upgrade_node # # Requires YunoHost version 2.7.12 or higher. -ynh_cron_upgrade_node() { +_ynh_cron_upgrade_node() { # Build the update script cat >"$n_install_dir/node_update.sh" <>"$phpfpm_path" fi - local finalphpconf="$fpm_config_dir/pool.d/$app.conf" - ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" + ynh_config_add --template="$phpfpm_path" --destination="$fpm_config_dir/pool.d/$app.conf" # Validate that the new php conf doesn't break php-fpm entirely if ! php-fpm${php_version} --test 2>/dev/null; then php-fpm${php_version} --test || true - ynh_safe_rm "$finalphpconf" + ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" ynh_die --message="The new configuration broke php-fpm?" fi ynh_systemd_action --service=$fpm_service --action=reload @@ -145,22 +146,21 @@ pm.process_idle_timeout = 10s # Remove the dedicated PHP-FPM config # -# usage: ynh_remove_fpm_config +# usage: ynh_config_remove_phpfpm # # Requires YunoHost version 2.7.2 or higher. -ynh_remove_fpm_config() { +ynh_config_remove_phpfpm() { local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) - local fpm_service=$(ynh_app_setting_get --key=fpm_service) ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" - ynh_systemd_action --service=$fpm_service --action=reload + ynh_systemd_action --service="php${php_version}-fpm" --action=reload } # Define the values to configure PHP-FPM # # [internal] # -# usage: ynh_get_scalable_phpfpm +# usage: _ynh_get_scalable_phpfpm # Footprint can be defined via the "fpm_footprint", to be set prior to calling this helper # low - Less than 20 MB of RAM by pool. # medium - Between 20 MB and 40 MB of RAM by pool. @@ -174,7 +174,7 @@ ynh_remove_fpm_config() { # medium - Low usage, few people or/and publicly accessible. # high - High usage, frequently visited website. # -ynh_get_scalable_phpfpm() { +_ynh_get_scalable_phpfpm() { # If no usage provided, default to the value existing in setting ... or to low local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) @@ -294,12 +294,12 @@ ynh_composer_exec() { # # The installed version is defined by $composer_version which should be defined as global prior to calling this helper # -# usage: ynh_install_composer [--workdir=$install_dir] [--install_args="--optimize-autoloader"] +# usage: ynh_composer_install [--workdir=$install_dir] [--install_args="--optimize-autoloader"] # | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. # | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include # # Requires YunoHost version 4.2 or higher. -ynh_install_composer() { +ynh_composer_install() { # ============ Argument parsing ============= local -A args_array=([w]=workdir= [a]=install_args=) local workdir @@ -309,7 +309,7 @@ ynh_install_composer() { install_args="${install_args:-}" # =========================================== - [[ -n "${composer_version}" ]] || ynh_die --message="\$composer_version should be defined before calling ynh_install_composer. (In the past, this was called \$YNH_COMPOSER_VERSION)" + [[ -n "${composer_version}" ]] || ynh_die --message="\$composer_version should be defined before calling ynh_composer_install. (In the past, this was called \$YNH_COMPOSER_VERSION)" [[ ! -e "$workdir/composer.phar" ]] || ynh_safe_rm $workdir/composer.phar diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index acc36ac66..780a45956 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -42,7 +42,7 @@ _ynh_load_ruby_in_path_and_other_tweaks() { # When not possible (e.g. in systemd service definition), please use direct path # to rbenv shims (e.g. $RBENV_ROOT/shims/bundle) # -# usage: ynh_install_ruby +# usage: ynh_ruby_install # # Adds the appropriate, specific version of ruby to the PATH variable (which # is also exported, to ease the use of ynh_exec_as_app). Also define variable @@ -50,9 +50,9 @@ _ynh_load_ruby_in_path_and_other_tweaks() { # (Environment="PATH=__PATH_WITH_RUBY__") # # Requires YunoHost version 3.2.2 or higher. -ynh_install_ruby () { +ynh_ruby_install () { - [[ -n "${ruby_version:-}" ]] || ynh_die --message="\$ruby_version should be defined prior to calling ynh_install_ruby" + [[ -n "${ruby_version:-}" ]] || ynh_die --message="\$ruby_version should be defined prior to calling ynh_ruby_install" # Load rbenv path in PATH local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" @@ -171,7 +171,7 @@ ynh_install_ruby () { rbenv alias $app $final_ruby_version # Cleanup Ruby versions - ynh_cleanup_ruby + _ynh_ruby_cleanup # Set environment for Ruby users echo "#rbenv @@ -183,15 +183,15 @@ eval \"\$(rbenv init -)\" # Load the environment eval "$(rbenv init -)" - _ynh_load_ruby_in_path_and_other_variable_tweaks + _ynh_load_ruby_in_path_and_other_tweaks } # Remove the version of Ruby used by the app. # # This helper will also cleanup Ruby versions # -# usage: ynh_remove_ruby -ynh_remove_ruby () { +# usage: ynh_ruby_remove +ynh_ruby_remove () { local ruby_version=$(ynh_app_setting_get --key=ruby_version) # Load rbenv path in PATH @@ -206,7 +206,7 @@ ynh_remove_ruby () { ynh_app_setting_delete --key=ruby_version # Cleanup Ruby versions - ynh_cleanup_ruby + _ynh_ruby_cleanup } # Remove no more needed versions of Ruby used by the app. @@ -214,9 +214,7 @@ ynh_remove_ruby () { # This helper will check what Ruby version are no more required, # and uninstall them # If no app uses Ruby, rbenv will be also removed. -# -# usage: ynh_cleanup_ruby -ynh_cleanup_ruby () { +_ynh_ruby_cleanup () { # List required Ruby versions local installed_apps=$(yunohost app list | grep -oP 'id: \K.*$') diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 56eb44428..a4128add3 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -2,17 +2,17 @@ # Create a dedicated systemd config # -# usage: ynh_add_systemd_config [--service=service] [--template=template] +# usage: ynh_config_add_systemd [--service=service] [--template=template] # | arg: -s, --service= - Service name (optionnal, `$app` by default) # | arg: -t, --template= - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template) # # This will use the template `../conf/.service`. # -# See the documentation of `ynh_add_config` for a description of the template +# See the documentation of `ynh_config_add` for a description of the template # format and how placeholders are replaced with actual variables. # # Requires YunoHost version 4.1.0 or higher. -ynh_add_systemd_config() { +ynh_config_add_systemd() { # ============ Argument parsing ============= local -A args_array=([s]=service= [t]=template=) local service @@ -22,7 +22,7 @@ ynh_add_systemd_config() { template="${template:-systemd.service}" # =========================================== - ynh_add_config --template="$template" --destination="/etc/systemd/system/$service.service" + ynh_config_add --template="$template" --destination="/etc/systemd/system/$service.service" systemctl enable $service --quiet systemctl daemon-reload @@ -30,11 +30,11 @@ ynh_add_systemd_config() { # Remove the dedicated systemd config # -# usage: ynh_remove_systemd_config [--service=service] +# usage: ynh_config_remove_systemd [--service=service] # | arg: -s, --service= - Service name (optionnal, $app by default) # # Requires YunoHost version 2.7.2 or higher. -ynh_remove_systemd_config() { +ynh_config_remove_systemd() { # ============ Argument parsing ============= local -A args_array=([s]=service=) local service @@ -117,7 +117,7 @@ ynh_systemd_action() { if [ -e "$log_path" ]; then tail --lines=$length "$log_path" >&2 fi - ynh_clean_check_starting + _ynh_clean_check_starting return 1 fi @@ -166,16 +166,11 @@ ynh_systemd_action() { tail --lines=$length "$log_path" >&2 fi fi - ynh_clean_check_starting + _ynh_clean_check_starting fi } -# Clean temporary process and file used by ynh_check_starting -# -# [internal] -# -# Requires YunoHost version 3.5.0 or higher. -ynh_clean_check_starting() { +_ynh_clean_check_starting() { if [ -n "${pid_tail:-}" ]; then # Stop the execution of tail. kill -SIGTERM $pid_tail 2>&1 diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index 64f9beb53..54517b3f5 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -2,15 +2,15 @@ # Create a dedicated config file from a template # -# usage: ynh_add_config --template="template" --destination="destination" +# usage: ynh_config_add --template="template" --destination="destination" # | arg: -t, --template= - Template config file to use # | arg: -d, --destination= - Destination of the config file # | arg: -j, --jinja - Use jinja template instead of legacy __MY_VAR__ # # examples: -# ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" -# ynh_add_config --jinja --template="config.j2" --destination="$install_dir/config" use the template file "../conf/config.j2" -# ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" +# ynh_config_add --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" +# ynh_config_add --jinja --template="config.j2" --destination="$install_dir/config" use the template file "../conf/config.j2" +# ynh_config_add --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" # ## ## How it works in "legacy" mode @@ -72,7 +72,7 @@ # into the app settings when configuration is done. # # Requires YunoHost version 4.1.0 or higher. -ynh_add_config() { +ynh_config_add() { # ============ Argument parsing ============= local -A args_array=([t]=template= [d]=destination= [j]=jinja) local template diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 72196394b..b638180e1 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -504,7 +504,7 @@ ynh_app_upgrading_from_version_before_or_equal_to() { } # Check if we should enforce sane default permissions (= disable rwx for 'others') -# on file/folders handled with ynh_setup_source and ynh_add_config +# on file/folders handled with ynh_setup_source and ynh_config_add # # [internal] # From 11e2b6d63cece297cf599b682651f62378dc5aa9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2024 23:19:19 +0200 Subject: [PATCH 073/146] helpers2.1: ynh_systemd_action '--line_match' -> '--wait_until' --- helpers/helpers.v2.1.d/fail2ban | 2 +- helpers/helpers.v2.1.d/mongodb | 2 +- helpers/helpers.v2.1.d/systemd | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index de3636f3b..be944e0e8 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -105,7 +105,7 @@ ignoreregex = chown -R "$app:$app" "/var/log/$app" chmod -R u=rwX,g=rX,o= "/var/log/$app" - ynh_systemd_action --service=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + ynh_systemd_action --service=fail2ban --action=reload --wait_until="(Started|Reloaded) Fail2Ban Service" --log_path=systemd local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 0eb9f4f8a..f2d05d844 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -241,7 +241,7 @@ ynh_install_mongo() { # Make sure MongoDB is started and enabled systemctl enable $mongodb_servicename --quiet systemctl daemon-reload --quiet - ynh_systemd_action --service=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" + ynh_systemd_action --service=$mongodb_servicename --action=restart --wait_until="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" # Integrate MongoDB service in YunoHost yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index a4128add3..64ef28224 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -53,10 +53,10 @@ ynh_config_remove_systemd() { # Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started # -# usage: ynh_systemd_action [--service=service] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] +# usage: ynh_systemd_action [--service=service] [--action=action] [ [--wait_until="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] # | arg: -n, --service= - Name of the service to start. Default : `$app` # | arg: -a, --action= - Action to perform with systemctl. Default: start -# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. +# | arg: -w, --wait_until= - The pattern to find in the log to attest the service is effectively fully started. # | arg: -p, --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` # | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. # | arg: -e, --length= - Length of the error log displayed for debugging : Default : 20 @@ -64,17 +64,17 @@ ynh_config_remove_systemd() { # Requires YunoHost version 3.5.0 or higher. ynh_systemd_action() { # ============ Argument parsing ============= - local -A args_array=([n]=service= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=) + local -A args_array=([n]=service= [a]=action= [w]=wait_until= [p]=log_path= [t]=timeout= [e]=length=) local service local action - local line_match + local wait_until local length local log_path local timeout ynh_handle_getopts_args "$@" service="${service:-$app}" action=${action:-start} - line_match=${line_match:-} + wait_until=${wait_until:-} length=${length:-20} log_path="${log_path:-/var/log/$service/$service.log}" timeout=${timeout:-300} @@ -86,7 +86,7 @@ ynh_systemd_action() { fi # Start to read the log - if [[ -n "$line_match" ]]; then + if [[ -n "$wait_until" ]]; then local templog="$(mktemp)" # Following the starting of the app in its log if [ "$log_path" == "systemd" ]; then @@ -121,8 +121,8 @@ ynh_systemd_action() { return 1 fi - # Start the timeout and try to find line_match - if [[ -n "${line_match:-}" ]]; then + # Start the timeout and try to find wait_until + if [[ -n "${wait_until:-}" ]]; then set +x local i=0 local starttime=$(date +%s) @@ -130,12 +130,12 @@ ynh_systemd_action() { # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout if [ "$log_path" == "systemd" ]; then # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action - if journalctl --unit=$service --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then + if journalctl --unit=$service --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$wait_until"; then ynh_print_info --message="The service $service has correctly executed the action ${action}." break fi else - if grep --extended-regexp --quiet "$line_match" "$templog"; then + if grep --extended-regexp --quiet "$wait_until" "$templog"; then ynh_print_info --message="The service $service has correctly executed the action ${action}." break fi From 6b6580a9194baf7b803eda67486e01a8eb9b98fb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2024 23:40:53 +0200 Subject: [PATCH 074/146] helpers2.1: further simplify logging helpers by removing the --message (ynh_script_progression too, no more args except the message) --- helpers/helpers.v2.1.d/apps | 14 +-- helpers/helpers.v2.1.d/apt | 8 +- helpers/helpers.v2.1.d/backup | 14 +-- helpers/helpers.v2.1.d/config | 32 +++---- helpers/helpers.v2.1.d/fail2ban | 6 +- helpers/helpers.v2.1.d/getopts | 6 +- helpers/helpers.v2.1.d/go | 18 ++-- helpers/helpers.v2.1.d/logging | 144 +++++++----------------------- helpers/helpers.v2.1.d/mongodb | 12 +-- helpers/helpers.v2.1.d/nodejs | 4 +- helpers/helpers.v2.1.d/php | 10 +-- helpers/helpers.v2.1.d/redis | 2 +- helpers/helpers.v2.1.d/ruby | 4 +- helpers/helpers.v2.1.d/string | 2 +- helpers/helpers.v2.1.d/systemd | 10 +-- helpers/helpers.v2.1.d/templating | 8 +- helpers/helpers.v2.1.d/user | 4 +- helpers/helpers.v2.1.d/utils | 30 +++---- 18 files changed, 124 insertions(+), 204 deletions(-) diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps index a3ee183a9..ca76caf8d 100644 --- a/helpers/helpers.v2.1.d/apps +++ b/helpers/helpers.v2.1.d/apps @@ -22,7 +22,7 @@ ynh_install_apps() { do # Retrieve the name of the app (part before ?) local one_app=$(cut -d "?" -f1 <<< "$one_app_and_its_args") - [ -z "$one_app" ] && ynh_die --message="You didn't provided a YunoHost app to install" + [ -z "$one_app" ] && ynh_die "You didn't provided a YunoHost app to install" yunohost tools update apps @@ -102,10 +102,10 @@ ynh_remove_apps() { if [[ -z "$required_by" ]] then # Remove $one_app - ynh_print_info --message="Removing of $one_app" + ynh_print_info "Removing of $one_app" yunohost app remove $one_app --purge else - ynh_print_info --message="$one_app was not removed because it's still required by${required_by}" + ynh_print_info "$one_app was not removed because it's still required by${required_by}" fi done fi @@ -134,7 +134,7 @@ ynh_spawn_app_shell() { # Force Bash to be used to run this helper if [[ ! $0 =~ \/?bash$ ]] then - ynh_print_warn --message="Please use Bash as shell" + ynh_print_warn "Please use Bash as shell" exit 1 fi @@ -142,13 +142,13 @@ ynh_spawn_app_shell() { local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id)) if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]] then - ynh_print_warn --message="$app is not in the apps list" + ynh_print_warn "$app is not in the apps list" exit 1 fi # Make sure the app has its own user if ! id -u "$app" &>/dev/null; then - ynh_print_warn --message="There is no \"$app\" system user" + ynh_print_warn "There is no \"$app\" system user" exit 1 fi @@ -156,7 +156,7 @@ ynh_spawn_app_shell() { local install_dir=$(ynh_app_setting_get --key=install_dir) if [ -z "$install_dir" ] then - ynh_print_warn --message="$app has no install_dir setting (does it use packaging format >=2?)" + ynh_print_warn "$app has no install_dir setting (does it use packaging format >=2?)" exit 1 fi diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 2cc1e33d1..71547abf6 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -49,7 +49,7 @@ ynh_apt_install_dependencies() { then # Cover a small edge case where a packager could have specified "php7.4-pwet php5-gni" which is confusing [[ $(echo $specific_php_version | wc -l) -eq 1 ]] \ - || ynh_die --message="Inconsistent php versions in dependencies ... found : $specific_php_version" + || ynh_die "Inconsistent php versions in dependencies ... found : $specific_php_version" dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common" @@ -142,12 +142,12 @@ EOF # Fake an install of those dependencies to see the errors # The sed command here is, Print only from 'Reading state info' to the end. [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 - ynh_die --message="Unable to install dependencies" + ynh_die "Unable to install dependencies" } rm --recursive --force "$TMPDIR" # Remove the temp dir. # check if the package is actually installed - _ynh_apt_package_is_installed "${app_ynh_deps}" || ynh_die --message="Unable to install dependencies" + _ynh_apt_package_is_installed "${app_ynh_deps}" || ynh_die "Unable to install dependencies" # Specific tweak related to Postgresql # -> trigger postgresql regenconf if we may have just installed postgresql @@ -291,7 +291,7 @@ _ynh_wait_dpkg_free() { # Check if the name of this file contains only numbers. if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then # If so, that a remaining of dpkg. - ynh_print_warn --message="dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + ynh_print_warn "dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." set -o xtrace # set -x return 1 fi diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index c0acbaa8d..b5047ddc5 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -41,9 +41,9 @@ ynh_backup() { # don't backup big data items if [[ "$is_data" == true ]] && ([[ ${do_not_backup_data:-0} -eq 1 ]] || [[ ${BACKUP_CORE_ONLY:-0} -eq 1 ]]); then if [ $BACKUP_CORE_ONLY -eq 1 ]; then - ynh_print_info --message="$target will not be saved, because 'BACKUP_CORE_ONLY' is set." + ynh_print_info "$target will not be saved, because 'BACKUP_CORE_ONLY' is set." else - ynh_print_info --message="$target will not be saved, because 'do_not_backup_data' is set." + ynh_print_info "$target will not be saved, because 'do_not_backup_data' is set." fi return 1 fi @@ -53,7 +53,7 @@ ynh_backup() { # ============================================================================== # Be sure the source path is not empty if [ ! -e "$target" ]; then - ynh_print_warn --message="File or folder '${target}' to be backed up does not exist" + ynh_print_warn "File or folder '${target}' to be backed up does not exist" return 1 fi @@ -67,7 +67,7 @@ ynh_backup() { # Check if dest_path already exists in tmp archive if [[ -e "${dest_path}" ]]; then - ynh_print_warn --message="Destination path '${dest_path}' already exist" + ynh_print_warn "Destination path '${dest_path}' already exist" return 1 fi @@ -146,7 +146,7 @@ ynh_restore() { if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then if [[ "$is_data" == true ]] then - ynh_print_info --message="Skipping $target which doesn't exists in the archive, probably because restoring from a safety-backup-before-upgrade" + ynh_print_info "Skipping $target which doesn't exists in the archive, probably because restoring from a safety-backup-before-upgrade" # Assume it's not a big deal, we may be restoring a safety-backup-before-upgrade which doesnt contain those return 0 else @@ -275,13 +275,13 @@ ynh_backup_if_checksum_is_different() { backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')" mkdir --parents "$(dirname "$backup_file_checksum")" cp --archive "$file" "$backup_file_checksum" # Backup the current file - ynh_print_warn --message="File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" echo "$backup_file_checksum" # Return the name of the backup file if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then local file_path_base64=$(echo "$file" | base64 -w0) if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64} then - ynh_print_warn --message="Diff with the original file:" + ynh_print_warn "Diff with the original file:" diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true fi fi diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index 317a2ce02..0988562b6 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -20,7 +20,7 @@ _ynh_app_config_get_one() { # Get value from app settings or from another file elif [[ "$type" == "file" ]]; then if [[ "$bind" == "settings" ]]; then - ynh_die --message="File '${short_setting}' can't be stored in settings" + ynh_die "File '${short_setting}' can't be stored in settings" fi old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" @@ -30,7 +30,7 @@ _ynh_app_config_get_one() { if [[ "$bind" == "settings" ]]; then old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" elif [[ "$bind" == *":"* ]]; then - ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" fi @@ -71,38 +71,38 @@ _ynh_app_config_apply_one() { # Save in a file elif [[ "$type" == "file" ]]; then if [[ "$bind" == "settings" ]]; then - ynh_die --message="File '${short_setting}' can't be stored in settings" + ynh_die "File '${short_setting}' can't be stored in settings" fi local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]]; then ynh_backup_if_checksum_is_different --file="$bind_file" ynh_safe_rm "$bind_file" ynh_delete_file_checksum --file="$bind_file" - ynh_print_info --message="File '$bind_file' removed" + ynh_print_info "File '$bind_file' removed" else ynh_backup_if_checksum_is_different --file="$bind_file" if [[ "${!short_setting}" != "$bind_file" ]]; then cp "${!short_setting}" "$bind_file" fi ynh_store_file_checksum --file="$bind_file" --update_only - ynh_print_info --message="File '$bind_file' overwritten with ${!short_setting}" + ynh_print_info "File '$bind_file' overwritten with ${!short_setting}" fi # Save value in app settings elif [[ "$bind" == "settings" ]]; then ynh_app_setting_set --key=$short_setting --value="${!short_setting}" - ynh_print_info --message="Configuration key '$short_setting' edited in app settings" + ynh_print_info "Configuration key '$short_setting' edited in app settings" # Save multiline text in a file elif [[ "$type" == "text" ]]; then if [[ "$bind" == *":"* ]]; then - ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" echo "${!short_setting}" >"$bind_file" ynh_store_file_checksum --file="$bind_file" --update_only - ynh_print_info --message="File '$bind_file' overwritten with the content provided in question '${short_setting}'" + ynh_print_info "File '$bind_file' overwritten with the content provided in question '${short_setting}'" # Set value into a kind of key/value file else @@ -121,7 +121,7 @@ _ynh_app_config_apply_one() { # We stored the info in settings in order to be able to upgrade the app ynh_app_setting_set --key=$short_setting --value="${!short_setting}" - ynh_print_info --message="Configuration key '$bind_key_' edited into $bind_file" + ynh_print_info "Configuration key '$bind_key_' edited into $bind_file" fi fi @@ -212,7 +212,7 @@ _ynh_app_config_show() { _ynh_app_config_validate() { # Change detection - ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1 + ynh_script_progression "Checking what changed in the new configuration..." local nothing_changed=true local changes_validated=true for short_setting in "${!old[@]}"; do @@ -248,12 +248,12 @@ _ynh_app_config_validate() { fi done if [[ "$nothing_changed" == "true" ]]; then - ynh_print_info --message="Nothing has changed" + ynh_print_info "Nothing has changed" exit 0 fi # Run validation if something is changed - ynh_script_progression --message="Validating the new configuration..." --weight=1 + ynh_script_progression "Validating the new configuration..." for short_setting in "${!old[@]}"; do [[ "${changed[$short_setting]}" == "false" ]] && continue @@ -320,7 +320,7 @@ ynh_app_action_run() { #ynh_return "result:" #ynh_return "$(echo "${result}" | sed 's/^/ /g')" else - ynh_die --message="No handler defined in app's script for action $1. If you are the maintainer of this app, you should define '$runner'" + ynh_die "No handler defined in app's script for action $1. If you are the maintainer of this app, you should define '$runner'" fi } @@ -339,14 +339,14 @@ ynh_app_config_run() { ;; apply) max_progression=4 - ynh_script_progression --message="Reading config panel description and current configuration..." + ynh_script_progression "Reading config panel description and current configuration..." ynh_app_config_get ynh_app_config_validate - ynh_script_progression --message="Applying the new configuration..." + ynh_script_progression "Applying the new configuration..." ynh_app_config_apply - ynh_script_progression --message="Configuration of $app completed" --last + ynh_script_progression "Configuration of $app completed" ;; *) ynh_app_action_run $1 diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index be944e0e8..f27a32626 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -65,7 +65,7 @@ ynh_config_add_fail2ban() { # If failregex is provided, Build a config file on-the-fly using $logpath and $failregex if [[ -n "${failregex:-}" ]]; then - test -n "$logpath" || ynh_die --message="ynh_config_add_fail2ban expects a logfile path as first argument and received nothing." + test -n "$logpath" || ynh_die "ynh_config_add_fail2ban expects a logfile path as first argument and received nothing." echo " [__APP__] @@ -109,8 +109,8 @@ ignoreregex = local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then - ynh_print_warn --message="Fail2ban failed to load the jail for $app" - ynh_print_warn --message="${fail2ban_error#*WARNING}" + ynh_print_warn "Fail2ban failed to load the jail for $app" + ynh_print_warn "${fail2ban_error#*WARNING}" fi } diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index 1e32bc982..a0ef13b13 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -100,9 +100,9 @@ ynh_handle_getopts_args() { getopts ":$getopts_parameters" parameter || true if [ "$parameter" = "?" ]; then - ynh_die --message="Invalid argument: -${OPTARG:-}" + ynh_die "Invalid argument: -${OPTARG:-}" elif [ "$parameter" = ":" ]; then - ynh_die --message="-$OPTARG parameter requires an argument." + ynh_die "-$OPTARG parameter requires an argument." else local shift_value=1 # Use the long option, corresponding to the short option read by getopts, as a variable @@ -128,7 +128,7 @@ ynh_handle_getopts_args() { # At this point, if all_args[0] start with "-", then the argument is not well formed if [ "${all_args[0]:0:1}" == "-" ]; then - ynh_die --message="Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?" + ynh_die "Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?" fi # Reduce the value of shift, because the option has been removed manually shift_value=$((shift_value - 1)) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index d28c396e9..499dd51be 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -3,7 +3,7 @@ ynh_go_try_bash_extension() { if [ -x src/configure ]; then src/configure && make -C src || { - ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." + ynh_print_info "Optional bash extension failed to build, but things will still work normally." } fi } @@ -53,7 +53,7 @@ _ynh_load_go_in_path_and_other_tweaks() { # Requires YunoHost version 3.2.2 or higher. ynh_go_install () { - [[ -n "${go_version:-}" ]] || ynh_die --message="\$go_version should be defined prior to calling ynh_go_install" + [[ -n "${go_version:-}" ]] || ynh_die "\$go_version should be defined prior to calling ynh_go_install" # Load goenv path in PATH local CLEAR_PATH="$goenv_install_dir/bin:$PATH" @@ -68,11 +68,11 @@ ynh_go_install () { mkdir -p $goenv_install_dir pushd "$goenv_install_dir" if ! [ -x "$goenv_install_dir/bin/goenv" ]; then - ynh_print_info --message="Downloading goenv..." + ynh_print_info "Downloading goenv..." git init -q git remote add origin https://github.com/syndbg/goenv.git else - ynh_print_info --message="Updating goenv..." + ynh_print_info "Updating goenv..." fi git fetch -q --tags --prune origin local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") @@ -85,11 +85,11 @@ ynh_go_install () { mkdir -p "$goenv_install_dir/plugins/xxenv-latest" pushd "$goenv_install_dir/plugins/xxenv-latest" if ! [ -x "$goenv_install_dir/plugins/xxenv-latest/bin/goenv-latest" ]; then - ynh_print_info --message="Downloading xxenv-latest..." + ynh_print_info "Downloading xxenv-latest..." git init -q git remote add origin https://github.com/momo-lab/xxenv-latest.git else - ynh_print_info --message="Updating xxenv-latest..." + ynh_print_info "Updating xxenv-latest..." fi git fetch -q --tags --prune origin local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") @@ -110,7 +110,7 @@ ynh_go_install () { # Install the requested version of Go local final_go_version=$(goenv latest --print "$go_version") - ynh_print_info --message="Installation of Go-$final_go_version" + ynh_print_info "Installation of Go-$final_go_version" goenv install --quiet --skip-existing "$final_go_version" 2>&1 # Store go_version into the config of this app @@ -181,7 +181,7 @@ _ynh_go_cleanup () { do if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1` then - ynh_print_info --message="Removing of Go-$installed_go_version" + ynh_print_info "Removing of Go-$installed_go_version" $goenv_install_dir/bin/goenv uninstall --force "$installed_go_version" fi done @@ -190,7 +190,7 @@ _ynh_go_cleanup () { if [[ ! $required_go_versions ]] then # Remove goenv environment configuration - ynh_print_info --message="Removing of goenv" + ynh_print_info "Removing of goenv" ynh_secure_remove --file="$goenv_install_dir" ynh_secure_remove --file="/etc/profile.d/goenv.sh" fi diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index 9993739a4..5c278ab0a 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -1,55 +1,25 @@ #!/bin/bash -# Print a message to stderr and exit +# Print a message to stderr and terminate the current script # -# usage: ynh_die --message=MSG [--ret_code=RETCODE] -# | arg: -m, --message= - Message to display -# | arg: -c, --ret_code= - Exit code to exit with -# -# Requires YunoHost version 2.4.0 or higher. +# usage: ynh_die "Some message" ynh_die() { - # ============ Argument parsing ============= - local -A args_array=([m]=message= [c]=ret_code=) - local message - local ret_code - ynh_handle_getopts_args "$@" - ret_code=${ret_code:-1} - # =========================================== - - echo "$message" 1>&2 - exit "$ret_code" + echo "$1" 1>&2 + exit 1 } -# Display a message in the 'INFO' logging category +# Print an "INFO" message # -# usage: ynh_print_info --message="Some message" -# | arg: -m, --message= - Message to display -# -# Requires YunoHost version 3.2.0 or higher. +# usage: ynh_print_info "Some message" ynh_print_info() { - # ============ Argument parsing ============= - local -A args_array=([m]=message=) - local message - ynh_handle_getopts_args "$@" - # =========================================== - - echo "$message" >&$YNH_STDINFO + echo "$1" >&$YNH_STDINFO } # Print a warning on stderr # -# usage: ynh_print_warn --message="Text to print" -# | arg: -m, --message= - The text to print -# -# Requires YunoHost version 3.2.0 or higher. +# usage: ynh_print_warn "Some message" ynh_print_warn() { - # ============ Argument parsing ============= - local -A args_array=([m]=message=) - local message - ynh_handle_getopts_args "$@" - # =========================================== - - echo -e "${message}" >&2 + echo "$1" >&2 } # Execute a command and redirect stderr to stdout @@ -82,6 +52,16 @@ ynh_exec_and_print_stderr_only_if_error() { fi } +# Return data to the YunoHost core for later processing +# (to be used by special hooks like app config panel and core diagnosis) +# +# usage: ynh_return somedata +# +# Requires YunoHost version 3.6.0 or higher. +ynh_return() { + echo "$1" >>"$YNH_STDRETURN" +} + # Initial definitions for ynh_script_progression increment_progression=0 previous_weight=0 @@ -92,107 +72,47 @@ progress_scale=20 progress_string2="####################" progress_string1="++++++++++++++++++++" progress_string0="...................." -# Define base_time when the file is sourced -base_time=$(date +%s) # Print a progress bar showing the progression of an app script # -# usage: ynh_script_progression --message=message [--weight=weight] [--time] -# | arg: -m, --message= - The text to print -# | arg: -w, --weight= - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script. -# | arg: -t, --time - Print the execution time since the last call to this helper. Especially usefull to define weights. The execution time is given for the duration since the previous call. So the weight should be applied to this previous call. -# | arg: -l, --last - Use for the last call of the helper, to fill the progression bar. +# usage: ynh_script_progression "Some message" # # Requires YunoHost version 3.5.0 or higher. ynh_script_progression() { set +o xtrace # set +x - # ============ Argument parsing ============= - local -A args_array=([m]=message= [w]=weight= [t]=time [l]=last) - local message - local weight - local time - local last - ynh_handle_getopts_args "$@" - weight=${weight:-1} - # =========================================== - - # Re-disable xtrace, ynh_handle_getopts_args set it back - set +o xtrace # set +x - - # Always activate time when running inside CI tests - if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then - time=${time:-1} - else - time=${time:-0} - fi - - last=${last:-0} - - # Get execution time since the last $base_time - local exec_time=$(($(date +%s) - $base_time)) - base_time=$(date +%s) - # Compute $max_progression (if we didn't already) if [ "$max_progression" = -1 ]; then # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented. - local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)" - # Get the number of call with a weight value - local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0) - - # Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight - local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]]*\).*/\1/g')" - # Get the weight of each occurrences of 'ynh_script_progression' in the script using -w - local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]]*\).*/\1/g')" - # Each value will be on a different line. - # Remove each 'end of line' and replace it by a '+' to sum the values. - local weight_values=$(($(echo "$weight_valuesA" "$weight_valuesB" | grep -v -E '^\s*$' | tr '\n' '+' | sed 's/+$/+0/g'))) - - # max_progression is a total number of calls to this helper. - # Less the number of calls with a weight value. - # Plus the total of weight values - max_progression=$(($helper_calls - $weight_calls + $weight_values)) + local helper_calls= + max_progression="$(grep --count "^[^#]*ynh_script_progression" $0)" fi # Increment each execution of ynh_script_progression in this script by the weight of the previous call. increment_progression=$(($increment_progression + $previous_weight)) # Store the weight of the current call in $previous_weight for next call - previous_weight=$weight + previous_weight=1 # Reduce $increment_progression to the size of the scale - if [ $last -eq 0 ]; then - local effective_progression=$(($increment_progression * $progress_scale / $max_progression)) + local effective_progression=$(($increment_progression * $progress_scale / $max_progression)) + # If last is specified, fill immediately the progression_bar - else - local effective_progression=$progress_scale - fi # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task # expected_progression is the progression expected after the current task local expected_progression="$((($increment_progression + $weight) * $progress_scale / $max_progression - $effective_progression))" - if [ $last -eq 1 ]; then - expected_progression=0 + + # Hack for the "--last" message + if grep -qw 'completed' <<< "$1"; + then + effective_progression=$progress_scale + expected_progression=0 fi # left_progression is the progression not yet done local left_progression="$(($progress_scale - $effective_progression - $expected_progression))" # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done. local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}" - local print_exec_time="" - if [ $time -eq 1 ] && [ "$exec_time" -gt 10 ]; then - print_exec_time=" [$(bc <<< "scale=1; $exec_time / 60" ) minutes]" - fi - - echo "[$progression_bar] > ${message}${print_exec_time}" >&$YNH_STDINFO + echo "[$progression_bar] > ${1}" >&$YNH_STDINFO set -o xtrace # set -x } - -# Return data to the YunoHost core for later processing -# (to be used by special hooks like app config panel and core diagnosis) -# -# usage: ynh_return somedata -# -# Requires YunoHost version 3.6.0 or higher. -ynh_return() { - echo "$1" >>"$YNH_STDRETURN" -} diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index f2d05d844..8f50ede59 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -206,7 +206,7 @@ ynh_mongo_remove_db() { if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists ynh_mongo_drop_db --database=$db_name # Remove the database else - ynh_print_warn --message="Database $db_name not found" + ynh_print_warn "Database $db_name not found" fi # Remove mongo user if it exists @@ -221,17 +221,17 @@ ynh_mongo_remove_db() { # ynh_install_mongo() { - [[ -n "${mongo_version:-}" ]] || ynh_die --message="\$mongo_version should be defined prior to calling ynh_install_mongo" + [[ -n "${mongo_version:-}" ]] || ynh_die "\$mongo_version should be defined prior to calling ynh_install_mongo" - ynh_print_info --message="Installing MongoDB Community Edition ..." + ynh_print_info "Installing MongoDB Community Edition ..." local mongo_debian_release=$YNH_DEBIAN_VERSION if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then - ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." + ynh_print_warn "Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." mongo_version="4.4" fi if [[ "$mongo_version" == "4.4" ]]; then - ynh_print_warn --message="Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release." + ynh_print_warn "Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release." mongo_debian_release=buster fi @@ -261,7 +261,7 @@ ynh_remove_mongo() { # Only remove the mongodb service if it is not installed. if ! _ynh_apt_package_is_installed "mongodb*" then - ynh_print_info --message="Removing MongoDB service..." + ynh_print_info "Removing MongoDB service..." mongodb_servicename=mongod # Remove the mongodb service yunohost service remove $mongodb_servicename diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index e47ab9eb0..f668e1675 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -44,7 +44,7 @@ _ynh_load_nodejs_in_path_and_other_tweaks() { ynh_nodejs_install() { # Use n, https://github.com/tj/n to manage the nodejs versions - [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_nodejs_install" + [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_install" # Create $n_install_dir mkdir --parents "$n_install_dir" @@ -111,7 +111,7 @@ ynh_nodejs_install() { # Requires YunoHost version 2.7.12 or higher. ynh_nodejs_remove() { - [[ -n "${nodejs_version:-}" ]] || ynh_die --message="\$nodejs_version should be defined prior to calling ynh_nodejs_remove" + [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_remove" # Remove the line for this app sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version" diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index 626bbe0ec..c453c7427 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -139,7 +139,7 @@ pm.process_idle_timeout = 10s if ! php-fpm${php_version} --test 2>/dev/null; then php-fpm${php_version} --test || true ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" - ynh_die --message="The new configuration broke php-fpm?" + ynh_die "The new configuration broke php-fpm?" fi ynh_systemd_action --service=$fpm_service --action=reload } @@ -213,7 +213,7 @@ _ynh_get_scalable_phpfpm() { elif [ "$usage" = "high" ]; then php_pm=static else - ynh_die --message="Does not recognize '$usage' as an usage value." + ynh_die "Does not recognize '$usage' as an usage value." fi # Get the total of RAM available, except swap. @@ -309,7 +309,7 @@ ynh_composer_install() { install_args="${install_args:-}" # =========================================== - [[ -n "${composer_version}" ]] || ynh_die --message="\$composer_version should be defined before calling ynh_composer_install. (In the past, this was called \$YNH_COMPOSER_VERSION)" + [[ -n "${composer_version}" ]] || ynh_die "\$composer_version should be defined before calling ynh_composer_install. (In the past, this was called \$YNH_COMPOSER_VERSION)" [[ ! -e "$workdir/composer.phar" ]] || ynh_safe_rm $workdir/composer.phar @@ -321,9 +321,9 @@ ynh_composer_install() { local out # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir/composer.phar $composer_url 2>&1) \ - || ynh_die --message="$out" + || ynh_die "$out" # install dependencies ynh_composer_exec --workdir="$workdir" --commands="install --no-dev $install_args" \ - || ynh_die --message="Unable to install core dependencies with Composer." + || ynh_die "Unable to install core dependencies with Composer." } diff --git a/helpers/helpers.v2.1.d/redis b/helpers/helpers.v2.1.d/redis index 545bb8705..310db411b 100644 --- a/helpers/helpers.v2.1.d/redis +++ b/helpers/helpers.v2.1.d/redis @@ -23,7 +23,7 @@ ynh_redis_get_free_db() { db=-1 done - test "$db" -eq -1 && ynh_die --message="No available Redis databases..." + test "$db" -eq -1 && ynh_die "No available Redis databases..." echo "$db" } diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index 780a45956..dae25c6fd 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -52,7 +52,7 @@ _ynh_load_ruby_in_path_and_other_tweaks() { # Requires YunoHost version 3.2.2 or higher. ynh_ruby_install () { - [[ -n "${ruby_version:-}" ]] || ynh_die --message="\$ruby_version should be defined prior to calling ynh_ruby_install" + [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_install" # Load rbenv path in PATH local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" @@ -252,7 +252,7 @@ _ynh_ruby_cleanup () { ynh_ruby_try_bash_extension() { if [ -x src/configure ]; then src/configure && make -C src 2>&1 || { - ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." + ynh_print_info "Optional bash extension failed to build, but things will still work normally." } fi } diff --git a/helpers/helpers.v2.1.d/string b/helpers/helpers.v2.1.d/string index 8a0fbab93..05c8a8cd0 100644 --- a/helpers/helpers.v2.1.d/string +++ b/helpers/helpers.v2.1.d/string @@ -128,7 +128,7 @@ ynh_sanitize_dbid() { ynh_normalize_url_path() { local path_url=$1 - test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." + test -n "$path_url" || ynh_die "ynh_normalize_url_path expect a URL path as first argument and received nothing." if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / path_url="/$path_url" # Add / at begin of path variable fi diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 64ef28224..99bf9c15a 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -131,12 +131,12 @@ ynh_systemd_action() { if [ "$log_path" == "systemd" ]; then # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action if journalctl --unit=$service --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$wait_until"; then - ynh_print_info --message="The service $service has correctly executed the action ${action}." + ynh_print_info "The service $service has correctly executed the action ${action}." break fi else if grep --extended-regexp --quiet "$wait_until" "$templog"; then - ynh_print_info --message="The service $service has correctly executed the action ${action}." + ynh_print_info "The service $service has correctly executed the action ${action}." break fi fi @@ -158,11 +158,11 @@ ynh_systemd_action() { echo "" >&2 fi if [ $i -eq $timeout ]; then - ynh_print_warn --message="The service $service didn't fully executed the action ${action} before the timeout." - ynh_print_warn --message="Please find here an extract of the end of the log of the service $service:" + ynh_print_warn "The service $service didn't fully executed the action ${action} before the timeout." + ynh_print_warn "Please find here an extract of the end of the log of the service $service:" journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service >&2 if [ -e "$log_path" ]; then - ynh_print_warn --message="\-\-\-" + ynh_print_warn "===" tail --lines=$length "$log_path" >&2 fi fi diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index 54517b3f5..208af1be8 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -88,7 +88,7 @@ ynh_config_add() { elif [ -f "$template" ]; then template_path=$template else - ynh_die --message="The provided template $template doesn't exist" + ynh_die "The provided template $template doesn't exist" fi ynh_backup_if_checksum_is_different --file="$destination" @@ -173,7 +173,7 @@ ynh_replace_vars() { # -v FOO tests if $FOO is defined # -v $FOO tests if ${!FOO} is defined # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 - [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" + [[ -v "${one_var:-}" ]] || ynh_die "Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" # Escape delimiter in match/replace string match_string="__${one_var^^}__" @@ -232,7 +232,7 @@ ynh_read_var_in_file() { after="${after:-}" # =========================================== - [[ -f $file ]] || ynh_die --message="File $file does not exists" + [[ -f $file ]] || ynh_die "File $file does not exists" set +o xtrace # set +x @@ -310,7 +310,7 @@ ynh_write_var_in_file() { after="${after:-}" # =========================================== - [[ -f $file ]] || ynh_die --message="File $file does not exists" + [[ -f $file ]] || ynh_die "File $file does not exists" set +o xtrace # set +x diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index 0510fd721..227653893 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -135,7 +135,7 @@ ynh_system_user_create() { else local shell="--shell /usr/sbin/nologin" fi - useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account" + useradd $user_home_dir --system --user-group $username $shell || ynh_die "Unable to create $username system account" fi local group @@ -163,7 +163,7 @@ ynh_system_user_delete() { if ynh_system_user_exists --username="$username"; then deluser $username else - ynh_print_warn --message="The user $username was not found" + ynh_print_warn "The user $username was not found" fi # Check if the group exists on the system diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index b638180e1..debdf1f90 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -166,8 +166,8 @@ ynh_setup_source() { local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" - [[ -n "$src_url" ]] || ynh_die --message="No URL defined for source $source_id$arch_prefix ?" - [[ -n "$src_sum" ]] || ynh_die --message="No sha256 sum defined for source $source_id$arch_prefix ?" + [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" if [[ -z "$src_format" ]] then @@ -195,7 +195,7 @@ ynh_setup_source() { if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] then - ynh_die --message="For source $source_id, expected either 'true' or 'false' for the extract parameter" + ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" fi # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... @@ -212,7 +212,7 @@ ynh_setup_source() { cp $local_src $src_filename fi - [ -n "$src_url" ] || ynh_die --message="Couldn't parse SOURCE_URL from $src_file_path ?" + [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" # If the file was prefetched but somehow doesn't match the sum, rm and redownload it if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status @@ -229,7 +229,7 @@ ynh_setup_source() { local out # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ - || ynh_die --message="$out" + || ynh_die "$out" fi # Check the control sum @@ -238,7 +238,7 @@ ynh_setup_source() { local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" local actual_size="$(du -hs ${src_filename} | cut --fields=1)" rm -f ${src_filename} - ynh_die --message="Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." + ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." fi fi @@ -302,7 +302,7 @@ ynh_setup_source() { if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then tar --extract --file=$src_filename --directory="$dest_dir" $strip else - ynh_die --message="Archive format unrecognized." + ynh_die "Archive format unrecognized." fi ynh_safe_rm "$src_filename" fi @@ -315,7 +315,7 @@ ynh_setup_source() { pushd "$dest_dir" for p in $patches_folder/${source_id}-*.patch; do echo $p - patch --strip=1 <$p || ynh_print_warn --message="Packagers /!\\ patch $p failed to apply" + patch --strip=1 <$p || ynh_print_warn "Packagers /!\\ patch $p failed to apply" done popd fi @@ -432,15 +432,15 @@ ynh_safe_rm() { set +o xtrace # set +x if [ $# -ge 2 ]; then - ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_safe_rm but it will be ignored... Use this helper with one argument at time." + ynh_print_warn "/!\ Packager ! You provided more than one argument to ynh_safe_rm but it will be ignored... Use this helper with one argument at time." fi if [[ -z "$target" ]]; then - ynh_print_warn --message="ynh_safe_rm called with empty argument, ignoring." + ynh_print_warn "ynh_safe_rm called with empty argument, ignoring." elif [[ ! -e $target ]]; then - ynh_print_info --message="'$target' wasn't deleted because it doesn't exist." + ynh_print_info "'$target' wasn't deleted because it doesn't exist." elif ! _acceptable_path_to_delete "$target"; then - ynh_print_warn --message="Not deleting '$target' because it is not an acceptable path to delete." + ynh_print_warn "Not deleting '$target' because it is not an acceptable path to delete." else rm --recursive "$target" fi @@ -486,7 +486,7 @@ ynh_app_upstream_version_changed() { # Requires YunoHost version 11.2 or higher. ynh_app_upgrading_from_version_before() { local version=$1 - [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" + [[ $version =~ '~ynh' ]] || ynh_die "Invalid argument for version, should include the ~ynhX prefix" dpkg --compare-versions $YNH_APP_CURRENT_VERSION lt $version } @@ -498,7 +498,7 @@ ynh_app_upgrading_from_version_before() { # Requires YunoHost version 11.2 or higher. ynh_app_upgrading_from_version_before_or_equal_to() { local version=$1 - [[ $version =~ '~ynh' ]] || ynh_die --message="Invalid argument for version, should include the ~ynhX prefix" + [[ $version =~ '~ynh' ]] || ynh_die "Invalid argument for version, should include the ~ynhX prefix" dpkg --compare-versions $YNH_APP_CURRENT_VERSION le $version } @@ -600,7 +600,7 @@ ynh_get_ram() { # =========================================== if [ $free -eq $total ]; then - ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram" + ynh_print_warn "You have to choose --free or --total when using ynh_get_ram" ram=0 elif [ $free -eq 1 ]; then local free_ram=$(LC_ALL=C vmstat --stats --unit M | grep "free memory" | awk '{print $1}') From b8a1a3a660222cb251b95d269b6d9f5164458850 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2024 23:45:37 +0200 Subject: [PATCH 075/146] helpers2.1: ynh_config_remove_systemd now uses positional --- helpers/helpers.v2.1.d/systemd | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 99bf9c15a..b7aa0508e 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -30,23 +30,14 @@ ynh_config_add_systemd() { # Remove the dedicated systemd config # -# usage: ynh_config_remove_systemd [--service=service] -# | arg: -s, --service= - Service name (optionnal, $app by default) -# -# Requires YunoHost version 2.7.2 or higher. +# usage: ynh_config_remove_systemd service +# | arg: service - Service name (optionnal, $app by default) ynh_config_remove_systemd() { - # ============ Argument parsing ============= - local -A args_array=([s]=service=) - local service - ynh_handle_getopts_args "$@" - local service="${service:-$app}" - # =========================================== - - local finalsystemdconf="/etc/systemd/system/$service.service" - if [ -e "$finalsystemdconf" ]; then + local service="${1:-$app}" + if [ -e "/etc/systemd/system/$service.service" ]; then ynh_systemd_action --service=$service --action=stop systemctl disable $service --quiet - ynh_safe_rm "$finalsystemdconf" + ynh_safe_rm "/etc/systemd/system/$service.service" systemctl daemon-reload fi } From e12d79390f1e99145de8e61101049aa2448a62e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 20 Jun 2024 11:15:19 +0200 Subject: [PATCH 076/146] resources/db: ensure dbtype is valid --- src/utils/resources.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/resources.py b/src/utils/resources.py index 21f74b191..8b35cdb88 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -1458,6 +1458,8 @@ class DatabaseAppResource(AppResource): db_helper_name = "mysql" elif self.dbtype == "postgresql": db_helper_name = "psql" + else: + raise RuntimeError(f"Invalid dbtype {self.dbtype}") self._run_script( "deprovision", f""" From d9b9aa1884ac356ee1df38819d909289110f11b6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 20 Jun 2024 13:44:09 +0200 Subject: [PATCH 077/146] helpers2.1: use positional args for file checksum helpers --- helpers/helpers.v2.1.d/apt | 2 +- helpers/helpers.v2.1.d/backup | 58 +++++++------------------------ helpers/helpers.v2.1.d/config | 25 ++++++++----- helpers/helpers.v2.1.d/go | 4 +-- helpers/helpers.v2.1.d/nginx | 6 ++-- helpers/helpers.v2.1.d/php | 2 +- helpers/helpers.v2.1.d/templating | 4 +-- helpers/helpers.v2.1.d/utils | 2 +- 8 files changed, 39 insertions(+), 64 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 71547abf6..520007e48 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -62,7 +62,7 @@ ynh_apt_install_dependencies() { if [[ -f "$old_php_finalphpconf" ]] then - ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf" + ynh_backup_if_checksum_is_different "$old_php_finalphpconf" ynh_remove_fpm_config fi fi diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index b5047ddc5..42f7476b0 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -201,34 +201,19 @@ ynh_restore_everything() { done } +_ynh_file_checksum_exists() { + local file=$1 + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + [[ -n "$(ynh_app_setting_get --key=$checksum_setting_name)" ]] +} + # Calculate and store a file checksum into the app settings # -# usage: ynh_store_file_checksum --file=file -# | arg: -f, --file= - The file on which the checksum will performed, then stored. -# -# $app should be defined when calling this helper -# -# Requires YunoHost version 2.6.4 or higher. +# usage: ynh_store_file_checksum /path/to/file ynh_store_file_checksum() { - # ============ Argument parsing ============= - local -A args_array=([f]=file= [u]=update_only) - local file - local update_only - update_only="${update_only:-0}" - ynh_handle_getopts_args "$@" - # =========================================== - + local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - # If update only, we don't save the new checksum if no old checksum exist - if [ $update_only -eq 1 ]; then - local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) - if [ -z "${checksum_value}" ]; then - unset backup_file_checksum - return 0 - fi - fi - ynh_app_setting_set --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then @@ -250,21 +235,12 @@ ynh_store_file_checksum() { # Verify the checksum and backup the file if it's different # -# usage: ynh_backup_if_checksum_is_different --file=file -# | arg: -f, --file= - The file on which the checksum test will be perfomed. -# | ret: the name of a backup file, or nothing +# usage: ynh_backup_if_checksum_is_different /path/to/file # # This helper is primarily meant to allow to easily backup personalised/manually # modified config files. -# -# Requires YunoHost version 2.6.4 or higher. ynh_backup_if_checksum_is_different() { - # ============ Argument parsing ============= - local -A args_array=([f]=file=) - local file - ynh_handle_getopts_args "$@" - # =========================================== - + local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum @@ -291,19 +267,9 @@ ynh_backup_if_checksum_is_different() { # Delete a file checksum from the app settings # -# usage: ynh_delete_file_checksum --file=file -# | arg: -f, --file= - The file for which the checksum will be deleted -# -# $app should be defined when calling this helper -# -# Requires YunoHost version 3.3.1 or higher. +# usage: ynh_delete_file_checksum /path/to/file ynh_delete_file_checksum() { - # ============ Argument parsing ============= - local -A args_array=([f]=file=) - local file - ynh_handle_getopts_args "$@" - # =========================================== - + local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' ynh_app_setting_delete --key=$checksum_setting_name } diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config index 0988562b6..73b5d85b3 100644 --- a/helpers/helpers.v2.1.d/config +++ b/helpers/helpers.v2.1.d/config @@ -75,16 +75,19 @@ _ynh_app_config_apply_one() { fi local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]]; then - ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_backup_if_checksum_is_different "$bind_file" ynh_safe_rm "$bind_file" - ynh_delete_file_checksum --file="$bind_file" + ynh_delete_file_checksum "$bind_file" ynh_print_info "File '$bind_file' removed" else - ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_backup_if_checksum_is_different "$bind_file" if [[ "${!short_setting}" != "$bind_file" ]]; then cp "${!short_setting}" "$bind_file" fi - ynh_store_file_checksum --file="$bind_file" --update_only + if _ynh_file_checksum_exists "$bind_file" + then + ynh_store_file_checksum "$bind_file" + fi ynh_print_info "File '$bind_file' overwritten with ${!short_setting}" fi @@ -99,9 +102,12 @@ _ynh_app_config_apply_one() { ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" - ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_backup_if_checksum_is_different "$bind_file" echo "${!short_setting}" >"$bind_file" - ynh_store_file_checksum --file="$bind_file" --update_only + if _ynh_file_checksum_exists "$bind_file" + then + ynh_store_file_checksum "$bind_file" + fi ynh_print_info "File '$bind_file' overwritten with the content provided in question '${short_setting}'" # Set value into a kind of key/value file @@ -115,9 +121,12 @@ _ynh_app_config_apply_one() { bind_key_=${bind_key_:-$short_setting} local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@${install_dir:-}@ | sed s/__APP__/$app/)" - ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_backup_if_checksum_is_different "$bind_file" ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}" - ynh_store_file_checksum --file="$bind_file" --update_only + if _ynh_file_checksum_exists "$bind_file" + then + ynh_store_file_checksum "$bind_file" + fi # We stored the info in settings in order to be able to upgrade the app ynh_app_setting_set --key=$short_setting --value="${!short_setting}" diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 499dd51be..22797397e 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -191,7 +191,7 @@ _ynh_go_cleanup () { then # Remove goenv environment configuration ynh_print_info "Removing of goenv" - ynh_secure_remove --file="$goenv_install_dir" - ynh_secure_remove --file="/etc/profile.d/goenv.sh" + ynh_safe_rm "$goenv_install_dir" + ynh_safe_rm "/etc/profile.d/goenv.sh" fi } diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index b3b2836ae..3eb6bf212 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -28,7 +28,7 @@ ynh_config_add_nginx() { ynh_replace --match="^#root_path_only" --replace="" --file="$finalnginxconf" fi - ynh_store_file_checksum --file="$finalnginxconf" + ynh_store_file_checksum "$finalnginxconf" ynh_systemd_action --service=nginx --action=reload } @@ -56,8 +56,8 @@ ynh_config_change_url_nginx() { # ynh_config_add inside ynh_config_add_nginx because the path may have # changed if we're changing the domain too...) local old_nginx_conf_path=/etc/nginx/conf.d/$old_domain.d/$app.conf - ynh_backup_if_checksum_is_different --file="$old_nginx_conf_path" - ynh_delete_file_checksum --file="$old_nginx_conf_path" + ynh_backup_if_checksum_is_different "$old_nginx_conf_path" + ynh_delete_file_checksum "$old_nginx_conf_path" ynh_safe_rm "$old_nginx_conf_path" # Regen the nginx conf diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index c453c7427..b47259ac5 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -76,7 +76,7 @@ ynh_config_add_phpfpm() { if [[ -f "$old_php_finalphpconf" ]] then - ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf" + ynh_backup_if_checksum_is_different "$old_php_finalphpconf" ynh_remove_fpm_config fi fi diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index 208af1be8..b7dc3bc4a 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -91,7 +91,7 @@ ynh_config_add() { ynh_die "The provided template $template doesn't exist" fi - ynh_backup_if_checksum_is_different --file="$destination" + ynh_backup_if_checksum_is_different "$destination" # Make sure to set the permissions before we copy the file # This is to cover a case where an attacker could have @@ -113,7 +113,7 @@ ynh_config_add() { ynh_replace_vars --file="$destination" fi - ynh_store_file_checksum --file="$destination" + ynh_store_file_checksum "$destination" } # Replace variables in a file diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index debdf1f90..3f985d227 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -422,7 +422,7 @@ _acceptable_path_to_delete() { fi } -# Remove a file or a directory securely +# Remove a file or a directory, checking beforehand that it's not a disastrous location to rm such as entire /var or /home # # usage: ynh_safe_rm path_to_remove # From f22c6ec3e9c0980fd91beeb4fd5a4b5ef730a98f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 20 Jun 2024 13:47:05 +0200 Subject: [PATCH 078/146] helpers2.1: ynh_systemd_action -> ynh_systemctl --- helpers/helpers.v2.1.d/fail2ban | 4 ++-- helpers/helpers.v2.1.d/mongodb | 2 +- helpers/helpers.v2.1.d/nginx | 4 ++-- helpers/helpers.v2.1.d/php | 4 ++-- helpers/helpers.v2.1.d/systemd | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index f27a32626..a382d5dce 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -105,7 +105,7 @@ ignoreregex = chown -R "$app:$app" "/var/log/$app" chmod -R u=rwX,g=rX,o= "/var/log/$app" - ynh_systemd_action --service=fail2ban --action=reload --wait_until="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + ynh_systemctl --service=fail2ban --action=reload --wait_until="(Started|Reloaded) Fail2Ban Service" --log_path=systemd local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then @@ -122,5 +122,5 @@ ignoreregex = ynh_config_remove_fail2ban() { ynh_safe_rm "/etc/fail2ban/jail.d/$app.conf" ynh_safe_rm "/etc/fail2ban/filter.d/$app.conf" - ynh_systemd_action --service=fail2ban --action=reload + ynh_systemctl --service=fail2ban --action=reload } diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 8f50ede59..370b88d92 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -241,7 +241,7 @@ ynh_install_mongo() { # Make sure MongoDB is started and enabled systemctl enable $mongodb_servicename --quiet systemctl daemon-reload --quiet - ynh_systemd_action --service=$mongodb_servicename --action=restart --wait_until="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" + ynh_systemctl --service=$mongodb_servicename --action=restart --wait_until="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" # Integrate MongoDB service in YunoHost yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index 3eb6bf212..59899600c 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -30,7 +30,7 @@ ynh_config_add_nginx() { ynh_store_file_checksum "$finalnginxconf" - ynh_systemd_action --service=nginx --action=reload + ynh_systemctl --service=nginx --action=reload } # Remove the dedicated nginx config @@ -40,7 +40,7 @@ ynh_config_add_nginx() { # Requires YunoHost version 2.7.2 or higher. ynh_config_remove_nginx() { ynh_safe_rm "/etc/nginx/conf.d/$domain.d/$app.conf" - ynh_systemd_action --service=nginx --action=reload + ynh_systemctl --service=nginx --action=reload } diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index b47259ac5..b72f16737 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -141,7 +141,7 @@ pm.process_idle_timeout = 10s ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" ynh_die "The new configuration broke php-fpm?" fi - ynh_systemd_action --service=$fpm_service --action=reload + ynh_systemctl --service=$fpm_service --action=reload } # Remove the dedicated PHP-FPM config @@ -153,7 +153,7 @@ ynh_config_remove_phpfpm() { local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" - ynh_systemd_action --service="php${php_version}-fpm" --action=reload + ynh_systemctl --service="php${php_version}-fpm" --action=reload } # Define the values to configure PHP-FPM diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index b7aa0508e..13b6cc293 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -35,7 +35,7 @@ ynh_config_add_systemd() { ynh_config_remove_systemd() { local service="${1:-$app}" if [ -e "/etc/systemd/system/$service.service" ]; then - ynh_systemd_action --service=$service --action=stop + ynh_systemctl --service=$service --action=stop systemctl disable $service --quiet ynh_safe_rm "/etc/systemd/system/$service.service" systemctl daemon-reload @@ -44,7 +44,7 @@ ynh_config_remove_systemd() { # Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started # -# usage: ynh_systemd_action [--service=service] [--action=action] [ [--wait_until="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] +# usage: ynh_systemctl [--service=service] [--action=action] [ [--wait_until="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] # | arg: -n, --service= - Name of the service to start. Default : `$app` # | arg: -a, --action= - Action to perform with systemctl. Default: start # | arg: -w, --wait_until= - The pattern to find in the log to attest the service is effectively fully started. @@ -53,7 +53,7 @@ ynh_config_remove_systemd() { # | arg: -e, --length= - Length of the error log displayed for debugging : Default : 20 # # Requires YunoHost version 3.5.0 or higher. -ynh_systemd_action() { +ynh_systemctl() { # ============ Argument parsing ============= local -A args_array=([n]=service= [a]=action= [w]=wait_until= [p]=log_path= [t]=timeout= [e]=length=) local service From b47aedc93286ae51ffe77ea71d7587e1406e25d3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 20 Jun 2024 14:02:56 +0200 Subject: [PATCH 079/146] helpers: remove god forsaken clumsy node_update cron job >_> --- helpers/helpers.v1.d/nodejs | 62 +---------------------------------- helpers/helpers.v2.1.d/nodejs | 60 --------------------------------- hooks/conf_regen/01-yunohost | 2 ++ 3 files changed, 3 insertions(+), 121 deletions(-) diff --git a/helpers/helpers.v1.d/nodejs b/helpers/helpers.v1.d/nodejs index be79cef66..de6d7a43e 100644 --- a/helpers/helpers.v1.d/nodejs +++ b/helpers/helpers.v1.d/nodejs @@ -83,7 +83,7 @@ ynh_use_nodejs() { # ynh_install_nodejs will install the version of node provided as argument by using n. # # usage: ynh_install_nodejs --nodejs_version=nodejs_version -# | arg: -n, --nodejs_version= - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0). The crontab will then handle the update of minor versions when needed. +# | arg: -n, --nodejs_version= - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0). # # `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use. # That's how it changes the version @@ -149,9 +149,6 @@ ynh_install_nodejs() { # Store nodejs_version into the config of this app ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version - # Build the update script and set the cronjob - ynh_cron_upgrade_node - ynh_use_nodejs } @@ -180,62 +177,5 @@ ynh_remove_nodejs() { ynh_secure_remove --file="$n_install_dir" ynh_secure_remove --file="/usr/local/n" sed --in-place "/N_PREFIX/d" /root/.bashrc - rm --force /etc/cron.daily/node_update fi } - -# Set a cron design to update your node versions -# -# [internal] -# -# This cron will check and update all minor node versions used by your apps. -# -# usage: ynh_cron_upgrade_node -# -# Requires YunoHost version 2.7.12 or higher. -ynh_cron_upgrade_node() { - # Build the update script - cat >"$n_install_dir/node_update.sh" <"/etc/cron.daily/node_update" <> $n_install_dir/node_update.log -EOF - - chmod +x "/etc/cron.daily/node_update" -} diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index f668e1675..e8eecb829 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -94,9 +94,6 @@ ynh_nodejs_install() { # Store nodejs_version into the config of this app ynh_app_setting_set --key=nodejs_version --value=$nodejs_version - # Build the update script and set the cronjob - _ynh_cron_upgrade_node - _ynh_load_nodejs_in_path_and_other_tweaks } @@ -126,62 +123,5 @@ ynh_nodejs_remove() { ynh_safe_rm "$n_install_dir" ynh_safe_rm "/usr/local/n" sed --in-place "/N_PREFIX/d" /root/.bashrc - rm --force /etc/cron.daily/node_update fi } - -# Set a cron design to update your node versions -# -# [internal] -# -# This cron will check and update all minor node versions used by your apps. -# -# usage: _ynh_cron_upgrade_node -# -# Requires YunoHost version 2.7.12 or higher. -_ynh_cron_upgrade_node() { - # Build the update script - cat >"$n_install_dir/node_update.sh" <"/etc/cron.daily/node_update" <> $n_install_dir/node_update.log -EOF - - chmod +x "/etc/cron.daily/node_update" -} diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 1d7a449e4..3d7bfa023 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -162,6 +162,8 @@ EOF mkdir -p ${pending_dir}/etc/dpkg/origins/ cp dpkg-origins ${pending_dir}/etc/dpkg/origins/yunohost + # Remove legacy hackish/clumsy nodejs autoupdate which ends up filling up space with ambiguous upgrades >_> + touch "/etc/cron.daily/node_update" } do_post_regen() { From eab36d069d10d761a752668b27d1e3f14a1a4759 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 20 Jun 2024 16:15:57 +0200 Subject: [PATCH 080/146] helper2.1: refactor composer helper again: workdir is $install_dir unless $composer_workdir is defined. ynh_composer_install only downloads composer. ynh_composer_exec runs composer command as $app unless composer_user=root is defined prior to calling the helper --- helpers/helpers.v2.1.d/php | 44 ++++++++++++++------------------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index b72f16737..dd252cd45 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -271,43 +271,35 @@ _ynh_get_scalable_phpfpm() { # Execute a command with Composer # -# usage: ynh_composer_exec [--workdir=$install_dir] --commands="commands" -# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir -# | arg: -c, --commands - Commands to execute. +# Will use $install_dir as workdir unless $composer_workdir exists (but that shouldnt be necessary) +# +# You may also define composer_user=root prior to call this helper if you absolutely need composer to run as root, but this is discouraged... +# +# usage: ynh_composer_exec commands # # Requires YunoHost version 4.2 or higher. ynh_composer_exec() { - # ============ Argument parsing ============= - local -A args_array=([w]=workdir= [c]=commands=) - local workdir - local commands - ynh_handle_getopts_args "$@" - workdir="${workdir:-${install_dir}}" - # =========================================== + local workdir="${composer_workdir:-$install_dir}" - COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ - php${php_version} "$workdir/composer.phar" $commands \ + COMPOSER_HOME="$workdir/.composer" \ + COMPOSER_MEMORY_LIMIT=-1 \ + sudo -E -u "${composer_user:-$app}" \ + php${php_version} "$workdir/composer.phar" $@ \ -d "$workdir" --no-interaction --no-ansi 2>&1 } # Install and initialize Composer in the given directory # -# The installed version is defined by $composer_version which should be defined as global prior to calling this helper +# The installed version is defined by $composer_version which should be defined +# as global prior to calling this helper. # -# usage: ynh_composer_install [--workdir=$install_dir] [--install_args="--optimize-autoloader"] -# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. -# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include +# Will use $install_dir as workdir unless $composer_workdir exists (but that shouldnt be necessary) +# +# usage: ynh_composer_install # # Requires YunoHost version 4.2 or higher. ynh_composer_install() { - # ============ Argument parsing ============= - local -A args_array=([w]=workdir= [a]=install_args=) - local workdir - local install_args - ynh_handle_getopts_args "$@" - workdir="${workdir:-$install_dir}" - install_args="${install_args:-}" - # =========================================== + local workdir="${composer_workdir:-$install_dir}" [[ -n "${composer_version}" ]] || ynh_die "\$composer_version should be defined before calling ynh_composer_install. (In the past, this was called \$YNH_COMPOSER_VERSION)" @@ -322,8 +314,4 @@ ynh_composer_install() { # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir/composer.phar $composer_url 2>&1) \ || ynh_die "$out" - - # install dependencies - ynh_composer_exec --workdir="$workdir" --commands="install --no-dev $install_args" \ - || ynh_die "Unable to install core dependencies with Composer." } From e0a9bafde203c8638eeba76b61f19c4d6b837795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 19 Jun 2024 15:14:25 +0200 Subject: [PATCH 081/146] helpers.v2.1: Add ynh_in_ci_tests to check if the scripts are running in CI or not --- helpers/helpers.v2.1.d/backup | 4 ++-- helpers/helpers.v2.1.d/utils | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 42f7476b0..abb41e041 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -216,7 +216,7 @@ ynh_store_file_checksum() { ynh_app_setting_set --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) - if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + if ynh_in_ci_tests; then # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ... local file_path_base64=$(echo "$file" | base64 -w0) mkdir -p /var/cache/yunohost/appconfbackup/ @@ -253,7 +253,7 @@ ynh_backup_if_checksum_is_different() { cp --archive "$file" "$backup_file_checksum" # Backup the current file ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" echo "$backup_file_checksum" # Return the name of the backup file - if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + if ynh_in_ci_tests; then local file_path_base64=$(echo "$file" | base64 -w0) if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64} then diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 3f985d227..e6eb2e88b 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -616,3 +616,14 @@ ynh_get_ram() { echo $ram } + +# Check if the scripts are being run by the package_check in CI +# +# usage: ynh_in_ci_tests +# +# Return 0 if in CI, 1 otherwise +# +# Requires YunoHost version 11.3 or higher. +ynh_in_ci_tests() { + [ "${PACKAGE_CHECK_EXEC:-0}" -eq 1 ] +} From 41b958113a26ec450b7ddc8f3128a99ce440590a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 20 Jun 2024 19:06:14 +0200 Subject: [PATCH 082/146] helpers2.1: unbound var @_@ --- helpers/helpers.v2.1.d/logging | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index 5c278ab0a..d10330581 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -100,7 +100,7 @@ ynh_script_progression() { # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task # expected_progression is the progression expected after the current task - local expected_progression="$((($increment_progression + $weight) * $progress_scale / $max_progression - $effective_progression))" + local expected_progression="$((($increment_progression + 1) * $progress_scale / $max_progression - $effective_progression))" # Hack for the "--last" message if grep -qw 'completed' <<< "$1"; From 6605df6eb270532435e84dddf3fa5e8c093e8ca3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 20 Jun 2024 19:06:54 +0200 Subject: [PATCH 083/146] helpers2.1: use the appropriate helper for apt provisioning/deprovisioning depending on the helpers_version in the manifest --- src/utils/resources.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 8b35cdb88..27bc05aee 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -155,6 +155,19 @@ class AppResource: elif manager and manager.current and "version" in manager.current: app_upstream_version = manager.current["version"].split("~")[0] + # FIXME : should use packaging.version to properly parse / compare versions >_> + self.helpers_version = None + if manager and manager.wanted and manager.wanted.get("integration", {}).get("helpers_version"): + self.helpers_version = manager.wanted.get("integration", {}).get("helpers_version") + elif manager and manager.current and manager.current.get("integration", {}).get("helpers_version"): + self.helpers_version = manager.current.get("integration", {}).get("helpers_version") + elif manager and manager.wanted and manager.wanted.get("packaging_format"): + self.helpers_version = str(manager.wanted.get("packaging_format")) + elif manager and manager.current and manager.current.get("packaging_format"): + self.helpers_version = str(manager.current.get("packaging_format")) + if not self.helpers_version: + self.helpers_version = "1" + replacements: dict[str, str] = { "__APP__": self.app, "__YNH_ARCH__": system_arch(), @@ -1182,11 +1195,19 @@ class AptDependenciesAppResource(AppResource): } def provision_or_update(self, context: Dict = {}): - script = " ".join(["ynh_install_app_dependencies", *self.packages]) + + if float(self.helpers_version) >= 2.1: + ynh_apt_install_dependencies = "ynh_apt_install_dependencies" + ynh_apt_install_dependencies_from_extra_repository = "ynh_apt_install_dependencies_from_extra_repository" + else: + ynh_apt_install_dependencies = "ynh_install_app_dependencies" + ynh_apt_install_dependencies_from_extra_repository = "ynh_install_extra_app_dependencies" + + script = " ".join([ynh_apt_install_dependencies, *self.packages]) for repo, values in self.extras.items(): script += "\n" + " ".join( [ - "ynh_install_extra_app_dependencies", + ynh_apt_install_dependencies_from_extra_repository, f"--repo='{values['repo']}'", f"--key='{values['key']}'", f"--package='{' '.join(values['packages'])}'", @@ -1197,7 +1218,12 @@ class AptDependenciesAppResource(AppResource): self._run_script("provision_or_update", script) def deprovision(self, context: Dict = {}): - self._run_script("deprovision", "ynh_remove_app_dependencies") + if float(self.helpers_version) >= 2.1: + ynh_apt_remove_dependencies = "ynh_apt_remove_dependencies" + else: + ynh_apt_remove_dependencies = "ynh_remove_app_dependencies" + + self._run_script("deprovision", ynh_apt_remove_dependencies) class PortsResource(AppResource): From b1b3c6eff89b6079433d90ca5b6b3eaf10c921c5 Mon Sep 17 00:00:00 2001 From: alexAubin <4533074+alexAubin@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:07:42 +0000 Subject: [PATCH 084/146] :art: Format Python code with Black --- src/utils/resources.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 0d2b1ddc9..b408fde47 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -157,10 +157,22 @@ class AppResource: # FIXME : should use packaging.version to properly parse / compare versions >_> self.helpers_version = None - if manager and manager.wanted and manager.wanted.get("integration", {}).get("helpers_version"): - self.helpers_version = manager.wanted.get("integration", {}).get("helpers_version") - elif manager and manager.current and manager.current.get("integration", {}).get("helpers_version"): - self.helpers_version = manager.current.get("integration", {}).get("helpers_version") + if ( + manager + and manager.wanted + and manager.wanted.get("integration", {}).get("helpers_version") + ): + self.helpers_version = manager.wanted.get("integration", {}).get( + "helpers_version" + ) + elif ( + manager + and manager.current + and manager.current.get("integration", {}).get("helpers_version") + ): + self.helpers_version = manager.current.get("integration", {}).get( + "helpers_version" + ) elif manager and manager.wanted and manager.wanted.get("packaging_format"): self.helpers_version = str(manager.wanted.get("packaging_format")) elif manager and manager.current and manager.current.get("packaging_format"): @@ -1198,10 +1210,14 @@ class AptDependenciesAppResource(AppResource): if float(self.helpers_version) >= 2.1: ynh_apt_install_dependencies = "ynh_apt_install_dependencies" - ynh_apt_install_dependencies_from_extra_repository = "ynh_apt_install_dependencies_from_extra_repository" + ynh_apt_install_dependencies_from_extra_repository = ( + "ynh_apt_install_dependencies_from_extra_repository" + ) else: ynh_apt_install_dependencies = "ynh_install_app_dependencies" - ynh_apt_install_dependencies_from_extra_repository = "ynh_install_extra_app_dependencies" + ynh_apt_install_dependencies_from_extra_repository = ( + "ynh_install_extra_app_dependencies" + ) script = " ".join([ynh_apt_install_dependencies, *self.packages]) for repo, values in self.extras.items(): @@ -1488,10 +1504,11 @@ class DatabaseAppResource(AppResource): raise RuntimeError(f"Invalid dbtype {self.dbtype}") self._run_script( - "deprovision", f""" + "deprovision", + f""" ynh_{db_helper_name}_database_exists "{db_name}" && ynh_{db_helper_name}_drop_db "{db_name}" || true ynh_{db_helper_name}_user_exists "{db_user}" && ynh_{db_helper_name}_drop_user "{db_user}" || true -""" +""", ) self.delete_setting("db_name") From 2f933b5bf25cd16d94a8a7504a360a0673f54cff Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 20 Jun 2024 21:38:11 +0200 Subject: [PATCH 085/146] Update changelog for 11.2.15 --- debian/changelog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index 2f9e295eb..f5059d42c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +yunohost (11.2.15) stable; urgency=low + + - apps: new experimentals "2.1" helpers ([#1855](http://github.com/YunoHost/yunohost/pull/1855)) + - apps: when removing an app with --purge, also remove /var/log/{app} + - apps: drop clumsy auto-update of nodejs via cron job which fills up disk space with nodejs copies and doesnt actually restart the app services... + - apps: fix apt resources when multiple extras are set ([#1869](http://github.com/YunoHost/yunohost/pull/1869)) + - mail: allow aliases for sender addresses of apps ([#1843](http://github.com/YunoHost/yunohost/pull/1843)) + + Thanks to all contributors <3 ! (alexAubin, Chris Vogel, Félix Piédallu) + + -- Alexandre Aubin Thu, 20 Jun 2024 21:20:47 +0200 + yunohost (11.2.14.1) stable; urgency=low - helpers: Fix typo in ynh_read_manifest documentation ([#1866](http://github.com/YunoHost/yunohost/pull/1866)) From a25033bba5404da07a715de117a7219c2d006362 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 Jun 2024 14:20:56 +0200 Subject: [PATCH 086/146] apps/logs: fix some information not being redacted because of the packaging v2 flows --- src/app.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 3051fbbf2..12d67f0bd 100644 --- a/src/app.py +++ b/src/app.py @@ -2992,19 +2992,31 @@ def _make_environment_for_app_script( # If packaging format v2, load all settings if manifest["packaging_format"] >= 2 or force_include_app_settings: env_dict["app"] = app + data_to_redact = [] + prefixes_or_suffixes_to_redact = ["pwd", "pass", "passwd", "password", "passphrase", "secret", "key", "token"] + for setting_name, setting_value in _get_app_settings(app).items(): # Ignore special internal settings like checksum__ # (not a huge deal to load them but idk...) if setting_name.startswith("checksum__"): continue - env_dict[setting_name] = str(setting_value) + setting_value = str(setting_value) + env_dict[setting_name] = setting_value + + # Check if we should redact this setting value + # (the check on the setting length exists to prevent stupid stuff like redacting empty string or something which is actually just 0/1, true/false, ... + if len(setting_value) > 6 and any(setting_name.startswith(p) or setting_name.endswith(p) for p in prefixes_or_suffixes_to_redact): + data_to_redact.append(setting_value) # Special weird case for backward compatibility... # 'path' was loaded into 'path_url' ..... if "path" in env_dict: env_dict["path_url"] = env_dict["path"] + for operation_logger in OperationLogger._instances: + operation_logger.data_to_redact.extend(data_to_redact) + return env_dict From 06c8fbc881333fabfcf6825ae6c48e417a2b6f7e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 Jun 2024 14:39:49 +0200 Subject: [PATCH 087/146] logs: misc ad-hoc tweaks to limit the noise in log sharing --- helpers/helpers.v2.1.d/apt | 38 ++++++++++++++++---------------- helpers/helpers.v2.1.d/backup | 4 ++++ helpers/helpers.v2.1.d/getopts | 7 ++++-- helpers/helpers.v2.1.d/php | 40 ++++++++++++++++++++++------------ helpers/helpers.v2.1.d/setting | 4 +++- helpers/helpers.v2.1.d/systemd | 4 ++-- helpers/helpers.v2.1.d/utils | 7 +++--- src/log.py | 15 +++++++++++++ 8 files changed, 77 insertions(+), 42 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 520007e48..a68b3b29d 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -16,15 +16,14 @@ YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true" # # Requires YunoHost version 2.6.4 or higher. ynh_apt_install_dependencies() { - local dependencies=$@ - # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) - dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" - local dependencies=${dependencies//|/ | } + + # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) + local dependencies="$(sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g' <<< "$@" | sed 's/|/ | /')" local version=$(ynh_read_manifest "version") local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps # Handle specific versions - if [[ "$dependencies" =~ [\<=\>] ]]; then + if grep '[<=>]' <<< "$dependencies"; then # Replace version specifications by relationships syntax # https://www.debian.org/doc/debian-policy/ch-relationships.html # Sed clarification @@ -33,7 +32,7 @@ ynh_apt_install_dependencies() { # \+ matches one or more occurence of the previous characters, for >= or >>. # [^,]\+ matches all characters except ',' # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)' - dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')" + dependencies="$(sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g' <<< "$dependencies")" fi # ############################## # @@ -43,7 +42,7 @@ ynh_apt_install_dependencies() { # Check for specific php dependencies which requires sury # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni" # The (?<=php) syntax corresponds to lookbehind ;) - local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u) + local specific_php_version=$(grep -oP '(?<=php)[0-9.]+(?=-|\>|)' <<< "$dependencies" | sort -u) if [[ -n "$specific_php_version" ]] then @@ -128,21 +127,22 @@ EOF # NB: this is in a subshell (though not sure why exactly not just use pushd/popd...) cd "$TMPDIR" # Install the fake package without its dependencies with dpkg --force-depends - LC_ALL=C equivs-build ./control 2>&1 - LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" 2>&1 | tee ./dpkg_log + LC_ALL=C equivs-build ./control > ./equivs_log 2>&1 || { cat ./equivs_log; false; } + LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" > ./dpkg_log 2>&1 ) # Then install the missing dependencies with apt install - _ynh_apt_install --fix-broken \ - || { # If the installation failed - # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) - # Parse the list of problematic dependencies from dpkg's log ... - # (relevant lines look like: "foo-ynh-deps depends on bar; however:") - local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')" - # Fake an install of those dependencies to see the errors - # The sed command here is, Print only from 'Reading state info' to the end. - [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 - ynh_die "Unable to install dependencies" + _ynh_apt_install --fix-broken || { + # If the installation failed + # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) + # Parse the list of problematic dependencies from dpkg's log ... + # (relevant lines look like: "foo-ynh-deps depends on bar; however:") + cat $TMPDIR/dpkg_log + local problematic_dependencies="$(grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' $TMPDIR/dpkg_log | tr '\n' ' ')" + # Fake an install of those dependencies to see the errors + # The sed command here is, Print only from 'Reading state info' to the end. + [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 + ynh_die "Unable to install dependencies" } rm --recursive --force "$TMPDIR" # Remove the temp dir. diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index abb41e041..c310cc0b8 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -211,6 +211,7 @@ _ynh_file_checksum_exists() { # # usage: ynh_store_file_checksum /path/to/file ynh_store_file_checksum() { + set +o xtrace # set +x local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' @@ -231,6 +232,7 @@ ynh_store_file_checksum() { fi # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. unset backup_file_checksum + set -o xtrace # set -x } # Verify the checksum and backup the file if it's different @@ -240,6 +242,7 @@ ynh_store_file_checksum() { # This helper is primarily meant to allow to easily backup personalised/manually # modified config files. ynh_backup_if_checksum_is_different() { + set +o xtrace # set +x local file=$1 local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) @@ -263,6 +266,7 @@ ynh_backup_if_checksum_is_different() { fi fi fi + set -o xtrace # set -x } # Delete a file checksum from the app settings diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index a0ef13b13..381c74256 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -46,10 +46,13 @@ # # Requires YunoHost version 3.2.2 or higher. ynh_handle_getopts_args() { + # Trick to only re-enable debugging if it was set before + local xtrace_enable=$(set +o | grep xtrace) + # Manage arguments only if there's some provided set +o xtrace # set +x if [ $# -eq 0 ]; then - set -o xtrace # set -x + eval "$xtrace_enable" return fi @@ -181,5 +184,5 @@ ynh_handle_getopts_args() { # Call parse_arg and pass the modified list of args as an array of arguments. parse_arg "${arguments[@]}" - set -o xtrace # set -x + eval "$xtrace_enable" } diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index dd252cd45..e8ff6d51d 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -176,6 +176,8 @@ ynh_config_remove_phpfpm() { # _ynh_get_scalable_phpfpm() { + set +o xtrace # set +x + # If no usage provided, default to the value existing in setting ... or to low local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) local usage=${fpm_usage_in_setting:-low} @@ -195,16 +197,6 @@ _ynh_get_scalable_phpfpm() { footprint=50 fi - # Define the factor to determine min_spare_servers - # to avoid having too few children ready to start for heavy apps - if [ $footprint -le 20 ]; then - min_spare_servers_factor=8 - elif [ $footprint -le 35 ]; then - min_spare_servers_factor=5 - else - min_spare_servers_factor=3 - fi - # Define the way the process manager handle child processes. if [ "$usage" = "low" ]; then php_pm=ondemand @@ -216,9 +208,6 @@ _ynh_get_scalable_phpfpm() { ynh_die "Does not recognize '$usage' as an usage value." fi - # Get the total of RAM available, except swap. - local max_ram=$(ynh_get_ram --total) - at_least_one() { # Do not allow value below 1 if [ $1 -le 0 ]; then @@ -228,10 +217,13 @@ _ynh_get_scalable_phpfpm() { fi } + # Get the total of RAM available, except swap. + local total_ram=$(ynh_get_ram --total) + # Define pm.max_children # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. # So if PHP-FPM start the maximum of children, it won't exceed half of the ram. - php_max_children=$(($max_ram / 2 / $footprint)) + php_max_children=$(($total_ram / 2 / $footprint)) # If process manager is set as static, use half less children. # Used as static, there's always as many children as the value of pm.max_children if [ "$php_pm" = "static" ]; then @@ -253,6 +245,17 @@ _ynh_get_scalable_phpfpm() { fi if [ "$php_pm" = "dynamic" ]; then + + # Define the factor to determine min_spare_servers + # to avoid having too few children ready to start for heavy apps + if [ $footprint -le 20 ]; then + min_spare_servers_factor=8 + elif [ $footprint -le 35 ]; then + min_spare_servers_factor=5 + else + min_spare_servers_factor=3 + fi + # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor)) php_min_spare_servers=$(at_least_one $php_min_spare_servers) @@ -267,6 +270,15 @@ _ynh_get_scalable_phpfpm() { php_max_spare_servers=0 php_start_servers=0 fi + + set -o xtrace # set -x + + # For debugging, since otherwise things are hidden with set +x/-x + echo "php_pm: $php_pm" + echo "php_max_children: $php_max_children" + echo "php_min_spare_servers: $php_min_spare_servers" + echo "php_max_spare_servers: $php_max_spare_servers" + echo "php_start_servers: $php_start_servers" } # Execute a command with Composer diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 0d737491e..84d755b3b 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -68,6 +68,8 @@ ynh_app_setting_delete() { # [internal] # ynh_app_setting() { + # Trick to only re-enable debugging if it was set before + local xtrace_enable=$(set +o | grep xtrace) set +o xtrace # set +x ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <&2 fi diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index e6eb2e88b..673ce6ba8 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -158,7 +158,6 @@ ynh_setup_source() { local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" - local src_sumprg="sha256sum" local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" src_in_subdir=${src_in_subdir:-true} @@ -215,7 +214,7 @@ ynh_setup_source() { [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" # If the file was prefetched but somehow doesn't match the sum, rm and redownload it - if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status then rm -f "$src_filename" fi @@ -233,9 +232,9 @@ ynh_setup_source() { fi # Check the control sum - if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status then - local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)" local actual_size="$(du -hs ${src_filename} | cut --fields=1)" rm -f ${src_filename} ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." diff --git a/src/log.py b/src/log.py index 818d0c1a9..45dc884fc 100755 --- a/src/log.py +++ b/src/log.py @@ -45,12 +45,18 @@ LOG_FILE_EXT = ".log" BORING_LOG_LINES = [ r"set [+-]x$", r"set [+-]o xtrace$", + r"\+ set \+o$", + r"\+ grep xtrace$", + r"local 'xtrace_enable=", r"set [+-]o errexit$", r"set [+-]o nounset$", r"trap '' EXIT", r"local \w+$", r"local exit_code=(1|0)$", r"local legacy_args=.*$", + r"local _globalapp=.*$", + r"local checksum_setting_name=.*$", + r"ynh_app_setting ", # (note the trailing space to match the "low level" one called by other setting helpers) r"local -A args_array$", r"args_array=.*$", r"ret_code=1", @@ -62,8 +68,17 @@ BORING_LOG_LINES = [ r"\[?\['? -n '' '?\]\]?$", r"rm -rf /var/cache/yunohost/download/$", r"type -t ynh_clean_setup$", + r"DEBUG - \+ unset \S+$", r"DEBUG - \+ echo '", + r"DEBUG - \+ LC_ALL=C$", + r"DEBUG - \+ DEBIAN_FRONTEND=noninteractive$", r"DEBUG - \+ exit (1|0)$", + r"DEBUG - \+ app=\S+$", + r"DEBUG - \+\+ app=\S+$", + r"DEBUG - \+\+ jq -r .\S+$", + r"DEBUG - \+\+ sed 's/\^null\$//'$", + "DEBUG - \\+ sed --in-place \$'s\\\\001", + "DEBUG - \\+ sed --in-place 's\u0001.*$", ] From dd8db1883ad7db242c92fc63f6bafbb09f865219 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 Jun 2024 14:57:19 +0200 Subject: [PATCH 088/146] helpers2.1: drop unused 'local source' mechanism from ynh_setup_source --- helpers/helpers.v2.1.d/utils | 7 ------- 1 file changed, 7 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 673ce6ba8..e6ad72eb1 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -197,9 +197,6 @@ ynh_setup_source() { ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" fi - # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... - local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" - # Gotta use this trick with 'dirname' because source_id may contain slashes x_x mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" @@ -207,10 +204,6 @@ ynh_setup_source() { if [ "$src_format" = "docker" ]; then src_platform="${src_platform:-"linux/$YNH_ARCH"}" else - if test -e "$local_src"; then - cp $local_src $src_filename - fi - [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" # If the file was prefetched but somehow doesn't match the sum, rm and redownload it From 1b5074d857ac22ca0dff1b35702305181eff93d6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 Jun 2024 18:29:59 +0200 Subject: [PATCH 089/146] helpers: add a new ynh_app_setting_set_default to replace the unecessarily complex 'if [ ${foo:-} ]' trick --- helpers/helpers.v1.d/setting | 36 ++++++++++++++++++++++++++++++++++ helpers/helpers.v2.1.d/setting | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/helpers/helpers.v1.d/setting b/helpers/helpers.v1.d/setting index 82a5d274e..c1946f6f8 100644 --- a/helpers/helpers.v1.d/setting +++ b/helpers/helpers.v1.d/setting @@ -52,6 +52,42 @@ ynh_app_setting_set() { fi } +# Set an application setting but only if the "$key" variable ain't set yet +# +# Note that it doesn't just define the setting but ALSO define the $foobar variable +# +# Hence it's meant as a replacement for this legacy overly complex syntax: +# +# if [ -z "${foo:-}" ] +# then +# foo="bar" +# ynh_app_setting_set --key="foo" --value="$foo" +# fi +# +# usage: ynh_app_setting_set_default --app=app --key=key --value=value +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting name to set +# | arg: -v, --value= - the default setting value to set +# +# Requires YunoHost version 11.1.16 or higher. +ynh_app_setting_set_default() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=akv + local -A args_array=([a]=app= [k]=key= [v]=value=) + local app + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [ -z "${!key:-}" ]; then + eval $key=\$value + ynh_app_setting "set" "$app" "$key" "$value" + fi +} + # Delete an application setting # # usage: ynh_app_setting_delete --app=app --key=key diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index 84d755b3b..d8053066d 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -42,6 +42,41 @@ ynh_app_setting_set() { ynh_app_setting "set" "$app" "$key" "$value" } +# Set an application setting but only if the "$key" variable ain't set yet +# +# Note that it doesn't just define the setting but ALSO define the $foobar variable +# +# Hence it's meant as a replacement for this legacy overly complex syntax: +# +# if [ -z "${foo:-}" ] +# then +# foo="bar" +# ynh_app_setting_set --key="foo" --value="$foo" +# fi +# +# usage: ynh_app_setting_set_default --app=app --key=key --value=value +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting name to set +# | arg: -v, --value= - the default setting value to set +# +# Requires YunoHost version 11.1.16 or higher. +ynh_app_setting_set_default() { + # ============ Argument parsing ============= + local _globalapp=${app-:} + local -A args_array=([a]=app= [k]=key= [v]=value=) + local app + local key + local value + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + # =========================================== + + if [ -z "${!key:-}" ]; then + eval $key=\$value + ynh_app_setting "set" "$app" "$key" "$value" + fi +} + # Delete an application setting # # usage: ynh_app_setting_delete --app=app --key=key From 0bd69e56228bf4a53436bec940990c4f66aa029f Mon Sep 17 00:00:00 2001 From: alexAubin <4533074+alexAubin@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:27:31 +0000 Subject: [PATCH 090/146] :art: Format Python code with Black --- src/app.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index 12d67f0bd..b729eab19 100644 --- a/src/app.py +++ b/src/app.py @@ -2993,7 +2993,16 @@ def _make_environment_for_app_script( if manifest["packaging_format"] >= 2 or force_include_app_settings: env_dict["app"] = app data_to_redact = [] - prefixes_or_suffixes_to_redact = ["pwd", "pass", "passwd", "password", "passphrase", "secret", "key", "token"] + prefixes_or_suffixes_to_redact = [ + "pwd", + "pass", + "passwd", + "password", + "passphrase", + "secret", + "key", + "token", + ] for setting_name, setting_value in _get_app_settings(app).items(): # Ignore special internal settings like checksum__ @@ -3006,7 +3015,10 @@ def _make_environment_for_app_script( # Check if we should redact this setting value # (the check on the setting length exists to prevent stupid stuff like redacting empty string or something which is actually just 0/1, true/false, ... - if len(setting_value) > 6 and any(setting_name.startswith(p) or setting_name.endswith(p) for p in prefixes_or_suffixes_to_redact): + if len(setting_value) > 6 and any( + setting_name.startswith(p) or setting_name.endswith(p) + for p in prefixes_or_suffixes_to_redact + ): data_to_redact.append(setting_value) # Special weird case for backward compatibility... From 3d728d90ceba004968947fc3b8bcff71c7609650 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 Jun 2024 18:32:22 +0200 Subject: [PATCH 091/146] helpers2.1: rework the fpm usage/footprint madness --- helpers/helpers.v2.1.d/apt | 7 +- helpers/helpers.v2.1.d/php | 286 +++++++++++-------------------------- 2 files changed, 86 insertions(+), 207 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index a68b3b29d..296e17b95 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -56,12 +56,9 @@ ynh_apt_install_dependencies() { # If the PHP version changed, remove the old fpm conf if [ -n "$old_php_version" ] && [ "$old_php_version" != "$specific_php_version" ]; then - local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) - local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" - - if [[ -f "$old_php_finalphpconf" ]] + if [[ -f "/etc/php/$php_version/fpm/pool.d/$app.conf" ]] then - ynh_backup_if_checksum_is_different "$old_php_finalphpconf" + ynh_backup_if_checksum_is_different "/etc/php/$php_version/fpm/pool.d/$app.conf" ynh_remove_fpm_config fi fi diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index e8ff6d51d..a05768015 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -19,89 +19,68 @@ fi # # usage: ynh_config_add_phpfpm # -# This helper assumes the app has an conf/extra_php-fpm.conf snippet +# This will automatically generate an appropriate PHP-FPM configuration for this app. # -# The actual PHP configuration will be automatically generated, -# and your extra_php-fpm.conf will be appended (typically contains PHP upload limits) +# The resulting configuration will be deployed to the appropriate place: +# /etc/php/$php_version/fpm/pool.d/$app.conf # -# The resulting configuration will be deployed to the appropriate place, /etc/php/$php_version/fpm/pool.d/$app.conf +# If the app provides a conf/extra_php-fpm.conf template, it will be appended +# to the generated configuration. (In the vast majority of cases, this shouldnt +# be necessary) # -# Performance-related options in the PHP conf, such as : -# pm.max_children, pm.start_servers, pm.min_spare_servers pm.max_spare_servers -# are computed from two parameters called "usage" and "footprint" which can be set to low/medium/high. (cf details below) +# $php_version should be defined prior to calling this helper, but there should +# be no reason to manually set it, as it is automatically set by the apt +# helpers/resources when installing phpX.Y dependencies (PHP apps should at +# least install phpX.Y-fpm using the apt helper/resource) # -# If you wish to tweak those, please initialize the settings `fpm_usage` and `fpm_footprint` -# *prior* to calling this helper. Otherwise, "low" will be used as a default for both values. +# $php_group can be defined as a global (from _common.sh) if the worker +# processes should run with a different group than $app # -# Otherwise, if you want the user to have control over these, we encourage to create a config panel -# (which should ultimately be standardized by the core ...) +# Additional "pm" and "php_admin_value" settings which are meant to be possibly +# configurable by admins from a future standard config panel at some point, +# related to performance and availability of the app, for which tweaking may be +# required if the app is used by "plenty" of users and other memory/CPU load +# considerations.... # -# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM. -# So it will be used to defined 'pm.max_children' -# A lower value for the footprint will allow more children for 'pm.max_children'. And so for -# 'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the -# value of 'pm.max_children' -# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores. +# If you have good reasons to be willing to use different +# defaults than the one set by this helper (while still allowing admin to +# override it) you should use `ynh_app_setting_set_default` # -# The usage value will defined the way php will handle the children for the pool. -# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the -# service is used, otherwise no child will stay alive. This config gives the lower footprint when the -# service is idle. But will use more proc since it has to start a child as soon it's used. -# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children -# equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request. -# The number of children can grow if needed. The footprint can stay low if the service is idle, but -# not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few -# children already available. -# Set as 'high', the process manager will be set at 'static'. There will be always as many children as -# 'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum -# RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many -# children ready to answer. +# - $php_upload_max_filezise: corresponds upload_max_filesize and post_max_size. Defaults to 50M +# - $php_process_management: corresponds to "pm" (ondemand, dynamic, static). Defaults to ondemand +# - $php_max_children: by default, computed from "total RAM" divided by 40, cf _default_php_max_children +# - $php_memory_limit: by default, 128M (from global php.ini) +# +# Note that if $php_process_management is set to "dynamic", then these +# variables MUST be defined prior to calling the helper (no default value) ... +# Check PHP-FPM's manual for more info on what these are (: ... +# +# - $php_start_servers +# - $php_min_spare_servers +# - $php_max_spare_servers # -# Requires YunoHost version 4.1.0 or higher. ynh_config_add_phpfpm() { - # ============ Argument parsing ============= - local -A args_array=([g]=group=) - local group - ynh_handle_getopts_args "$@" - group=${group:-} - # =========================================== - # If the PHP version changed, remove the old fpm conf - # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper) - # FIXME: so is this still needed @_@ - local old_php_version=$(ynh_app_setting_get --key=php_version) - if [ -n "$old_php_version" ] && [ "$old_php_version" != "$php_version" ]; then - local old_php_fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) - local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" + [[ -n "${php_version:-}" ]] || ynh_die "\$php_version should be defined prior to calling ynh_config_add_phpfpm. You should not need to define it manually, it is automatically set by the apt helper when installing the phpX.Y- depenencies" - if [[ -f "$old_php_finalphpconf" ]] - then - ynh_backup_if_checksum_is_different "$old_php_finalphpconf" - ynh_remove_fpm_config - fi - fi + # Apps may define $php_group as a global (e.g. from _common.sh) to change this + # (this is not meant to be overridable by users) + local php_group=${php_group:-$app} - local fpm_service="php${php_version}-fpm" - local fpm_config_dir="/etc/php/$php_version/fpm" + # Meant to be overridable by users from a standard config panel at some point ... + # Apps willing to tweak these should use ynh_setting_set_default_value (in install and upgrade?) + # + local php_upload_max_filesize=${php_upload_max_filesize:-50M} + local php_process_management=${php_process_management:-ondemand} # alternatively 'dynamic' or 'static' + local php_max_children=${php_max_children:-$(_default_php_max_children)} + local php_memory_limit=${php_memory_limit:-128M} # default value is from global php.ini - # Create the directory for FPM pools - mkdir --parents "$fpm_config_dir/pool.d" - - # FIXME: zzzz do we really need those ... - ynh_app_setting_set --key=fpm_config_dir --value="$fpm_config_dir" - ynh_app_setting_set --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --key=php_version --value=$php_version - - # Define the values to use for the configuration of PHP. - _ynh_get_scalable_phpfpm - - local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app") - local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" - echo " + local phpfpm_template=$(mktemp) + cat << EOF > $phpfpm_template [__APP__] user = __APP__ -group = __PHPFPM_GROUP__ +group = __PHP_GROUP__ chdir = __INSTALL_DIR__ @@ -109,39 +88,48 @@ listen = /var/run/php/php__PHP_VERSION__-fpm-__APP__.sock listen.owner = www-data listen.group = www-data -pm = __PHP_PM__ +pm = __PHP_PROCESS_MANAGEMENT__ pm.max_children = __PHP_MAX_CHILDREN__ pm.max_requests = 500 request_terminate_timeout = 1d -" >"$phpfpm_path" - if [ "$php_pm" = "dynamic" ]; then - echo " +EOF + if [ "$php_process_management" = "dynamic" ]; then + cat << EOF >> $phpfpm_template pm.start_servers = __PHP_START_SERVERS__ pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__ pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__ -" >>"$phpfpm_path" - - elif [ "$php_pm" = "ondemand" ]; then - echo " +EOF + elif [ "$php_process_management" = "ondemand" ]; then + cat << EOF >> $phpfpm_template pm.process_idle_timeout = 10s -" >>"$phpfpm_path" +EOF fi - # Concatene the extra config. + cat << EOF >> $phpfpm_template +php_admin_value[upload_max_filesize] = __PHP_UPLOAD_MAX_FILESIZE__ +php_admin_value[post_max_size] = __PHP_UPLOAD_MAX_FILESIZE__ +php_admin_value[memory_limit] = __PHP_MEMORY_LIMIT__ +EOF + + # Concatene the extra config if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then - cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path" + cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_template" fi - ynh_config_add --template="$phpfpm_path" --destination="$fpm_config_dir/pool.d/$app.conf" + # Make sure the fpm pool dir exists + mkdir --parents "/etc/php/$php_version/fpm/pool.d" + # And hydrate configuration + ynh_config_add --template="$phpfpm_template" --destination="/etc/php/$php_version/fpm/pool.d/$app.conf" # Validate that the new php conf doesn't break php-fpm entirely if ! php-fpm${php_version} --test 2>/dev/null; then php-fpm${php_version} --test || true - ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" + ynh_safe_rm "/etc/php/$php_version/fpm/pool.d/$app.conf" ynh_die "The new configuration broke php-fpm?" fi - ynh_systemctl --service=$fpm_service --action=reload + + ynh_systemctl --service=php${php_version}-fpm --action=reload } # Remove the dedicated PHP-FPM config @@ -150,137 +138,31 @@ pm.process_idle_timeout = 10s # # Requires YunoHost version 2.7.2 or higher. ynh_config_remove_phpfpm() { - local fpm_config_dir=$(ynh_app_setting_get --key=fpm_config_dir) - - ynh_safe_rm "$fpm_config_dir/pool.d/$app.conf" + ynh_safe_rm "/etc/php/$php_version/fpm/pool.d/$app.conf" ynh_systemctl --service="php${php_version}-fpm" --action=reload } -# Define the values to configure PHP-FPM -# -# [internal] -# -# usage: _ynh_get_scalable_phpfpm -# Footprint can be defined via the "fpm_footprint", to be set prior to calling this helper -# low - Less than 20 MB of RAM by pool. -# medium - Between 20 MB and 40 MB of RAM by pool. -# high - More than 40 MB of RAM by pool. -# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value. -# To have this value, use the following command and stress the service. -# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP -# -# Usage can be defined via the "fpm_usage", to be set prior to calling this helper -# low - Personal usage, behind the SSO. -# medium - Low usage, few people or/and publicly accessible. -# high - High usage, frequently visited website. -# -_ynh_get_scalable_phpfpm() { - - set +o xtrace # set +x - - # If no usage provided, default to the value existing in setting ... or to low - local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage) - local usage=${fpm_usage_in_setting:-low} - ynh_app_setting_set --key=fpm_usage --value=$usage - - # If no footprint provided, default to the value existing in setting ... or to low - local fpm_footprint_in_setting=$(ynh_app_setting_get --key=fpm_footprint) - local footprint=${fpm_footprint_in_setting:-low} - ynh_app_setting_set --key=fpm_footprint --value=$footprint - - # Set all characters as lowercase - if [ "$footprint" = "low" ]; then - footprint=20 - elif [ "$footprint" = "medium" ]; then - footprint=35 - elif [ "$footprint" = "high" ]; then - footprint=50 - fi - - # Define the way the process manager handle child processes. - if [ "$usage" = "low" ]; then - php_pm=ondemand - elif [ "$usage" = "medium" ]; then - php_pm=dynamic - elif [ "$usage" = "high" ]; then - php_pm=static - else - ynh_die "Does not recognize '$usage' as an usage value." - fi - - at_least_one() { - # Do not allow value below 1 - if [ $1 -le 0 ]; then - echo 1 - else - echo $1 - fi - } - - # Get the total of RAM available, except swap. +_default_php_max_children() { + # Get the total of RAM available local total_ram=$(ynh_get_ram --total) - # Define pm.max_children - # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. - # So if PHP-FPM start the maximum of children, it won't exceed half of the ram. - php_max_children=$(($total_ram / 2 / $footprint)) - # If process manager is set as static, use half less children. - # Used as static, there's always as many children as the value of pm.max_children - if [ "$php_pm" = "static" ]; then - php_max_children=$(($php_max_children / 2)) - fi - php_max_children=$(at_least_one $php_max_children) - + # The value of pm.max_children is the total amount of ram divide by 2, + # divide again by 20MB (= a default, classic worker footprint) This is + # designed such that if PHP-FPM start the maximum of children, it won't + # exceed half of the ram. + local php_max_children="$(($total_ram / 40))" + # Make sure we get at least max_children = 1 + if [ $php_max_children -le 0 ]; then + php_max_children=1 # To not overload the proc, limit the number of children to 4 times the number of cores. - local core_number=$(nproc) - local max_proc=$(($core_number * 4)) - if [ $php_max_children -gt $max_proc ]; then - php_max_children=$max_proc + elif [ $php_max_children -gt "$(($(nproc) * 4))" ]; then + php_max_children="$(($(nproc) * 4))" fi - # Get a potential forced value for php_max_children - local php_forced_max_children=$(ynh_app_setting_get --key=php_forced_max_children) - if [ -n "$php_forced_max_children" ]; then - php_max_children=$php_forced_max_children - fi - - if [ "$php_pm" = "dynamic" ]; then - - # Define the factor to determine min_spare_servers - # to avoid having too few children ready to start for heavy apps - if [ $footprint -le 20 ]; then - min_spare_servers_factor=8 - elif [ $footprint -le 35 ]; then - min_spare_servers_factor=5 - else - min_spare_servers_factor=3 - fi - - # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager - php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor)) - php_min_spare_servers=$(at_least_one $php_min_spare_servers) - - php_max_spare_servers=$(($php_max_children / 2)) - php_max_spare_servers=$(at_least_one $php_max_spare_servers) - - php_start_servers=$(($php_min_spare_servers + ($php_max_spare_servers - $php_min_spare_servers) / 2)) - php_start_servers=$(at_least_one $php_start_servers) - else - php_min_spare_servers=0 - php_max_spare_servers=0 - php_start_servers=0 - fi - - set -o xtrace # set -x - - # For debugging, since otherwise things are hidden with set +x/-x - echo "php_pm: $php_pm" - echo "php_max_children: $php_max_children" - echo "php_min_spare_servers: $php_min_spare_servers" - echo "php_max_spare_servers: $php_max_spare_servers" - echo "php_start_servers: $php_start_servers" + echo "$php_max_children" } + # Execute a command with Composer # # Will use $install_dir as workdir unless $composer_workdir exists (but that shouldnt be necessary) From e558513609c65fb96d9f77e38663697809846310 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 22 Jun 2024 22:52:31 +0200 Subject: [PATCH 092/146] helpers2.1: fix positional arg parsing in ynh_psql_create_user --- helpers/helpers.v2.1.d/postgresql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index 713743ec0..ffaeafbf3 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -81,6 +81,8 @@ ynh_psql_dump_db() { # | arg: pwd - the password to identify user by # ynh_psql_create_user() { + local user=$1 + local pwd=$2 sudo --login --user=postgres psql <<< "CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" } From 31ae5d1eaaff3e1865f80740de225c510b514eb4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 01:05:39 +0200 Subject: [PATCH 093/146] Misc fixes for flake8/mypy --- src/log.py | 2 +- src/utils/resources.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/log.py b/src/log.py index 45dc884fc..7950fca78 100755 --- a/src/log.py +++ b/src/log.py @@ -77,7 +77,7 @@ BORING_LOG_LINES = [ r"DEBUG - \+\+ app=\S+$", r"DEBUG - \+\+ jq -r .\S+$", r"DEBUG - \+\+ sed 's/\^null\$//'$", - "DEBUG - \\+ sed --in-place \$'s\\\\001", + "DEBUG - \\+ sed --in-place \\$'s\\\\001", "DEBUG - \\+ sed --in-place 's\u0001.*$", ] diff --git a/src/utils/resources.py b/src/utils/resources.py index b408fde47..c0a7404e2 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -156,29 +156,29 @@ class AppResource: app_upstream_version = manager.current["version"].split("~")[0] # FIXME : should use packaging.version to properly parse / compare versions >_> - self.helpers_version = None + self.helpers_version: float = 0 if ( manager and manager.wanted and manager.wanted.get("integration", {}).get("helpers_version") ): - self.helpers_version = manager.wanted.get("integration", {}).get( + self.helpers_version = float(manager.wanted.get("integration", {}).get( "helpers_version" - ) + )) elif ( manager and manager.current and manager.current.get("integration", {}).get("helpers_version") ): - self.helpers_version = manager.current.get("integration", {}).get( + self.helpers_version = float(manager.current.get("integration", {}).get( "helpers_version" - ) + )) elif manager and manager.wanted and manager.wanted.get("packaging_format"): - self.helpers_version = str(manager.wanted.get("packaging_format")) + self.helpers_version = float(manager.wanted.get("packaging_format")) elif manager and manager.current and manager.current.get("packaging_format"): - self.helpers_version = str(manager.current.get("packaging_format")) + self.helpers_version = float(manager.current.get("packaging_format")) if not self.helpers_version: - self.helpers_version = "1" + self.helpers_version = 1.0 replacements: dict[str, str] = { "__APP__": self.app, @@ -1208,7 +1208,7 @@ class AptDependenciesAppResource(AppResource): def provision_or_update(self, context: Dict = {}): - if float(self.helpers_version) >= 2.1: + if self.helpers_version >= 2.1: ynh_apt_install_dependencies = "ynh_apt_install_dependencies" ynh_apt_install_dependencies_from_extra_repository = ( "ynh_apt_install_dependencies_from_extra_repository" @@ -1234,7 +1234,7 @@ class AptDependenciesAppResource(AppResource): self._run_script("provision_or_update", script) def deprovision(self, context: Dict = {}): - if float(self.helpers_version) >= 2.1: + if self.helpers_version >= 2.1 ynh_apt_remove_dependencies = "ynh_apt_remove_dependencies" else: ynh_apt_remove_dependencies = "ynh_remove_app_dependencies" From ecb82ec6934f8e83b189b09683126643815c9fe8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 01:14:41 +0200 Subject: [PATCH 094/146] Oopsies --- src/utils/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index c0a7404e2..de05eee9c 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -1234,7 +1234,7 @@ class AptDependenciesAppResource(AppResource): self._run_script("provision_or_update", script) def deprovision(self, context: Dict = {}): - if self.helpers_version >= 2.1 + if self.helpers_version >= 2.1: ynh_apt_remove_dependencies = "ynh_apt_remove_dependencies" else: ynh_apt_remove_dependencies = "ynh_remove_app_dependencies" From 18092db1c856fc126ca8857818b909150a59520e Mon Sep 17 00:00:00 2001 From: alexAubin <4533074+alexAubin@users.noreply.github.com> Date: Sat, 22 Jun 2024 23:15:07 +0000 Subject: [PATCH 095/146] :art: Format Python code with Black --- src/utils/resources.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index de05eee9c..3e41044ea 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -162,17 +162,17 @@ class AppResource: and manager.wanted and manager.wanted.get("integration", {}).get("helpers_version") ): - self.helpers_version = float(manager.wanted.get("integration", {}).get( - "helpers_version" - )) + self.helpers_version = float( + manager.wanted.get("integration", {}).get("helpers_version") + ) elif ( manager and manager.current and manager.current.get("integration", {}).get("helpers_version") ): - self.helpers_version = float(manager.current.get("integration", {}).get( - "helpers_version" - )) + self.helpers_version = float( + manager.current.get("integration", {}).get("helpers_version") + ) elif manager and manager.wanted and manager.wanted.get("packaging_format"): self.helpers_version = float(manager.wanted.get("packaging_format")) elif manager and manager.current and manager.current.get("packaging_format"): From f26565d6dec15b2e4a363edcb0c70c1dc4ee0072 Mon Sep 17 00:00:00 2001 From: xabirequejo Date: Sat, 8 Jun 2024 10:26:01 +0000 Subject: [PATCH 096/146] Translated using Weblate (Basque) Currently translated at 100.0% (783 of 783 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/eu/ --- locales/eu.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/eu.json b/locales/eu.json index c21a18795..2b84b0763 100644 --- a/locales/eu.json +++ b/locales/eu.json @@ -601,8 +601,8 @@ "restore_hook_unavailable": "'{part}'-(e)rako lehengoratze agindua ez dago erabilgarri ez sisteman ezta fitxategian ere", "restore_cleaning_failed": "Ezin izan dira lehengoratzeko behin-behineko fitxategiak ezabatu", "restore_confirm_yunohost_installed": "Ziur al zaude dagoeneko instalatuta dagoen sistema lehengoratu nahi duzula? [{answers}]", - "restore_may_be_not_enough_disk_space": "Badirudi zure sistemak ez duela nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasuneko tartea: {margin} B)", - "restore_not_enough_disk_space": "Ez dago nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasuneko tartea: {margin} B)", + "restore_may_be_not_enough_disk_space": "Badirudi zure sistemak ez duela nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasun-tartea: {margin} B)", + "restore_not_enough_disk_space": "Ez dago nahikoa espazio (erabilgarri: {free_space} B, beharrezkoa {needed_space} B, segurtasun-tartea: {margin} B)", "restore_running_hooks": "Lehengoratzeko 'hook'ak exekutatzen…", "restore_system_part_failed": "Ezinezkoa izan da sistemaren '{part}' atala lehengoratzea", "server_reboot": "Zerbitzaria berrabiaraziko da", From 6ee40ac06acf5964e41f010e8b2c5019267b64ee Mon Sep 17 00:00:00 2001 From: Jose Riha Date: Tue, 11 Jun 2024 19:31:47 +0000 Subject: [PATCH 097/146] Translated using Weblate (Slovak) Currently translated at 31.2% (245 of 783 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/sk/ --- locales/sk.json | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/locales/sk.json b/locales/sk.json index a8af9df99..961555f8a 100644 --- a/locales/sk.json +++ b/locales/sk.json @@ -265,5 +265,18 @@ "service_description_rspamd": "Filtruje spam a iné funkcie týkajúce sa e-mailu", "log_letsencrypt_cert_renew": "Obnoviť '{}' certifikát Let's Encrypt", "domain_config_cert_summary_selfsigned": "UPOZORNENIE: Aktuálny certifikát je vlastnoručne podpísaný. Prehliadače budú návštevníkom zobrazovať strašidelné varovanie!", - "global_settings_setting_ssowat_panel_overlay_enabled": "Povoliť malú štvorcovú ikonu portálu „YunoHost“ na aplikáciach" + "global_settings_setting_ssowat_panel_overlay_enabled": "Povoliť malú štvorcovú ikonu portálu „YunoHost“ na aplikáciach", + "domain_config_mail_out": "Odchádzajúce e-maily", + "domain_config_default_app": "Predvolená aplikácia", + "domain_config_xmpp_help": "Pozor: niektoré funkcie XMPP vyžadujú aktualizáciu vašich DNS záznamov a obnovenie Lets Encrypt certifikátu pred tým, ako je ich možné zapnúť", + "domain_config_default_app_help": "Návštevníci budú pri návšteve tejto domény automaticky presmerovaní na túto doménu. Ak nenastavíte žiadnu aplikáciu, zobrazí sa stránka s prihlasovacím formulárom na portál.", + "registrar_infos": "Informácie o registrátorovi", + "domain_dns_registrar_managed_in_parent_domain": "Táto doména je subdoména {parent_domain_link}. Nastavenie DNS registrátora je spravovaná v konfiguračnom paneli {parent_domain}.", + "log_letsencrypt_cert_install": "Inštalovať certifikát Let's Encrypt na doménu '{}'", + "domain_config_cert_no_checks": "Ignorovať kontroly diagnostiky", + "domain_config_cert_install": "Nainštalovať certifikát Let's Encrypt", + "domain_config_mail_in": "Prichádzajúce e-maily", + "domain_config_cert_summary": "Stav certifikátu", + "domain_config_xmpp": "Krátke správy (XMPP)", + "log_app_makedefault": "Nastaviť '{}' ako predvolenú aplikáciu" } From 3942ea12aea990fae9e10217b8f7839aa957bb4b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 14:08:05 +0200 Subject: [PATCH 098/146] helpers2.1: fix ynh_config_add_logrotate when no arg is passed --- helpers/helpers.v2.1.d/logrotate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index 5fce19d0b..cd10e29d6 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -14,7 +14,7 @@ FIRST_CALL_TO_LOGROTATE="true" # Requires YunoHost version 2.6.4 or higher. ynh_config_add_logrotate() { - logfile="$1" + local logfile="${1:-}" set -o noglob if [[ -z "$logfile" ]]; then From d4857834f34075f4a61a023943a53d751c30ecfa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 14:13:17 +0200 Subject: [PATCH 099/146] helpers2.1: sudo -u$app -> sudo -u $app --- helpers/helpers.v2.1.d/user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index 227653893..0732258e6 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -178,5 +178,5 @@ ynh_system_user_delete() { # # usage: ynh_exec_as_app COMMAND [ARG ...] ynh_exec_as_app() { - sudo -E -u"$app" "$@" + sudo -E -u "$app" "$@" } From 262453f1324a2c6c507ce7adb8faaf5534d70569 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 14:27:37 +0200 Subject: [PATCH 100/146] helpers2.1: change default timeout of ynh_systemctl to 60s instead of 300s --- helpers/helpers.v2.1.d/systemd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 0efe515e9..b33ca5252 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -49,7 +49,7 @@ ynh_config_remove_systemd() { # | arg: -a, --action= - Action to perform with systemctl. Default: start # | arg: -w, --wait_until= - The pattern to find in the log to attest the service is effectively fully started. # | arg: -p, --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` -# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. +# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 60 seconds. # | arg: -e, --length= - Length of the error log displayed for debugging : Default : 20 # # Requires YunoHost version 3.5.0 or higher. @@ -68,7 +68,7 @@ ynh_systemctl() { wait_until=${wait_until:-} length=${length:-20} log_path="${log_path:-/var/log/$service/$service.log}" - timeout=${timeout:-300} + timeout=${timeout:-60} # =========================================== # Manage case of service already stopped From 9298738d062c88a3eae3b2f0454e0607470bf533 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 14:30:57 +0200 Subject: [PATCH 101/146] helpers2.1: display 100 lines instead of 20 in CI context when service fails to start --- helpers/helpers.v2.1.d/systemd | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index b33ca5252..259f25862 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -71,6 +71,12 @@ ynh_systemctl() { timeout=${timeout:-60} # =========================================== + # On CI, use length=100 because it's sometime hell to debug otherwise for super-long output + if ynh_in_ci_tests && [ $length -le 20 ] + then + length=100 + fi + # Manage case of service already stopped if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service; then return 0 From b3409729a6dafa75276bab975669855392bbfaa7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 14:36:55 +0200 Subject: [PATCH 102/146] helpers2.1: when using ynh_systemctl to reload/start/restart a service with a wait_until and it timesout, handle it as a filure rather than keep going --- helpers/helpers.v2.1.d/systemd | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 259f25862..0c0be3e67 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -162,6 +162,13 @@ ynh_systemctl() { ynh_print_warn "===" tail --lines=$length "$log_path" >&2 fi + + # If we tried to reload/start/restart the service but systemctl consider it to be still inactive/broken, then handle it as a failure + if ([ "$action" == "reload" ] || [ "$action" == "start" ] || [ "$action" == "restart" ]) && ! systemctl --quiet is-active $service + then + _ynh_clean_check_starting + return 1 + fi fi _ynh_clean_check_starting fi From 5f6df6a859f0328546d7553fa68fe69e9102f5b7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 14:49:11 +0200 Subject: [PATCH 103/146] helpers2.1: for some reason sudo -E doesn't preserve PATH even though it's exported, so we gotta explicitly use --preserve-env=PATH --- helpers/helpers.v2.1.d/user | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user index 0732258e6..1ce0e78de 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/user @@ -174,9 +174,9 @@ ynh_system_user_delete() { # Execute a command after sudoing as $app # -# Note that exported bash env variables are kept (using -E option of sudo) +# Note that the $PATH variable is preserved (using --preserve-env=PATH) # # usage: ynh_exec_as_app COMMAND [ARG ...] ynh_exec_as_app() { - sudo -E -u "$app" "$@" + sudo --preserve-env=PATH -u "$app" "$@" } From 2b1f74268f1a52a420459d11607c11461da528e2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 14:57:35 +0200 Subject: [PATCH 104/146] helpers2.1: var rename / cosmetic etc for nodejs/ruby/go version and install directories --- helpers/helpers.v2.1.d/go | 43 ++++++++++++----------- helpers/helpers.v2.1.d/nodejs | 44 ++++++++++++------------ helpers/helpers.v2.1.d/ruby | 64 +++++++++++++++++------------------ 3 files changed, 75 insertions(+), 76 deletions(-) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 22797397e..68f5ceaad 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -8,19 +8,18 @@ ynh_go_try_bash_extension() { fi } -goenv_install_dir="/opt/goenv" -go_version_path="$goenv_install_dir/versions" +readonly GOENV_INSTALL_DIR="/opt/goenv" # goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable. -export GOENV_ROOT="$goenv_install_dir" +export GOENV_ROOT="$GOENV_INSTALL_DIR" _ynh_load_go_in_path_and_other_tweaks() { # Get the absolute path of this version of go - local go_path="$go_version_path/$app/bin" + go_dir="$GOENV_INSTALL_DIR/versions/$app/bin" # Load the path of this version of go in $PATH - if [[ :$PATH: != *":$go_path"* ]]; then - PATH="$go_path:$PATH" + if [[ :$PATH: != *":$go_dir"* ]]; then + PATH="$go_dir:$PATH" fi # Export PATH such that it's available through sudo -E / ynh_exec_as $app @@ -32,7 +31,7 @@ _ynh_load_go_in_path_and_other_tweaks() { # Sets the local application-specific go version pushd ${install_dir} - $goenv_install_dir/bin/goenv local $go_version + $GOENV_INSTALL_DIR/bin/goenv local $go_version popd } @@ -56,7 +55,7 @@ ynh_go_install () { [[ -n "${go_version:-}" ]] || ynh_die "\$go_version should be defined prior to calling ynh_go_install" # Load goenv path in PATH - local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Go prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -65,9 +64,9 @@ ynh_go_install () { test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv # Install or update goenv - mkdir -p $goenv_install_dir - pushd "$goenv_install_dir" - if ! [ -x "$goenv_install_dir/bin/goenv" ]; then + mkdir -p $GOENV_INSTALL_DIR + pushd "$GOENV_INSTALL_DIR" + if ! [ -x "$GOENV_INSTALL_DIR/bin/goenv" ]; then ynh_print_info "Downloading goenv..." git init -q git remote add origin https://github.com/syndbg/goenv.git @@ -78,13 +77,13 @@ ynh_go_install () { local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") git checkout -q "$git_latest_tag" ynh_go_try_bash_extension - goenv=$goenv_install_dir/bin/goenv + goenv=$GOENV_INSTALL_DIR/bin/goenv popd # Install or update xxenv-latest - mkdir -p "$goenv_install_dir/plugins/xxenv-latest" - pushd "$goenv_install_dir/plugins/xxenv-latest" - if ! [ -x "$goenv_install_dir/plugins/xxenv-latest/bin/goenv-latest" ]; then + mkdir -p "$GOENV_INSTALL_DIR/plugins/xxenv-latest" + pushd "$GOENV_INSTALL_DIR/plugins/xxenv-latest" + if ! [ -x "$GOENV_INSTALL_DIR/plugins/xxenv-latest/bin/goenv-latest" ]; then ynh_print_info "Downloading xxenv-latest..." git init -q git remote add origin https://github.com/momo-lab/xxenv-latest.git @@ -97,10 +96,10 @@ ynh_go_install () { popd # Enable caching - mkdir -p "${goenv_install_dir}/cache" + mkdir -p "${GOENV_INSTALL_DIR}/cache" # Create shims directory if needed - mkdir -p "${goenv_install_dir}/shims" + mkdir -p "${GOENV_INSTALL_DIR}/shims" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH @@ -122,8 +121,8 @@ ynh_go_install () { # Set environment for Go users echo "#goenv -export GOENV_ROOT=$goenv_install_dir -export PATH=\"$goenv_install_dir/bin:$PATH\" +export GOENV_ROOT=$GOENV_INSTALL_DIR +export PATH=\"$GOENV_INSTALL_DIR/bin:$PATH\" eval \"\$(goenv init -)\" #goenv" > /etc/profile.d/goenv.sh @@ -142,7 +141,7 @@ ynh_go_remove () { local go_version=$(ynh_app_setting_get --key="go_version") # Load goenv path in PATH - local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Go prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -182,7 +181,7 @@ _ynh_go_cleanup () { if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1` then ynh_print_info "Removing of Go-$installed_go_version" - $goenv_install_dir/bin/goenv uninstall --force "$installed_go_version" + $GOENV_INSTALL_DIR/bin/goenv uninstall --force "$installed_go_version" fi done @@ -191,7 +190,7 @@ _ynh_go_cleanup () { then # Remove goenv environment configuration ynh_print_info "Removing of goenv" - ynh_safe_rm "$goenv_install_dir" + ynh_safe_rm "$GOENV_INSTALL_DIR" ynh_safe_rm "/etc/profile.d/goenv.sh" fi } diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index e8eecb829..5e5b13f4e 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -1,18 +1,17 @@ #!/bin/bash -n_install_dir="/opt/node_n" -node_version_path="$n_install_dir/n/versions/node" +readonly N_INSTALL_DIR="/opt/node_n" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. -export N_PREFIX="$n_install_dir" +export N_PREFIX="$N_INSTALL_DIR" _ynh_load_nodejs_in_path_and_other_tweaks() { # Get the absolute path of this version of node - local nodejs_path="$node_version_path/$nodejs_version/bin" + nodejs_dir="$N_INSTALL_DIR/n/versions/node/$nodejs_version/bin" # Load the path of this version of node in $PATH - if [[ :$PATH: != *":$nodejs_path"* ]]; then - PATH="$nodejs_path:$PATH" + if [[ :$PATH: != *":$nodejs_dir"* ]]; then + PATH="$nodejs_dir:$PATH" fi # Export PATH such that it's available through sudo -E / ynh_exec_as $app @@ -46,11 +45,11 @@ ynh_nodejs_install() { [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_install" - # Create $n_install_dir - mkdir --parents "$n_install_dir" + # Create $N_INSTALL_DIR + mkdir --parents "$N_INSTALL_DIR" # Load n path in PATH - CLEAR_PATH="$n_install_dir/bin:$PATH" + CLEAR_PATH="$N_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of node prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -59,10 +58,10 @@ ynh_nodejs_install() { test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n # Install (or update if YunoHost vendor/ folder updated since last install) n - mkdir -p $n_install_dir/bin/ - cp "$YNH_HELPERS_DIR/vendor/n/n" $n_install_dir/bin/n + mkdir -p $N_INSTALL_DIR/bin/ + cp "$YNH_HELPERS_DIR/vendor/n/n" $N_INSTALL_DIR/bin/n # Tweak for n to understand it's installed in $N_PREFIX - ynh_replace --match="^N_PREFIX=\${N_PREFIX-.*}$" --replace="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --file="$n_install_dir/bin/n" + ynh_replace --match="^N_PREFIX=\${N_PREFIX-.*}$" --replace="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --file="$N_INSTALL_DIR/bin/n" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH @@ -80,16 +79,18 @@ ynh_nodejs_install() { fi # Find the last "real" version for this major version of node. - real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1) + real_nodejs_version=$(find $N_INSTALL_DIR/n/versions/node/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1) real_nodejs_version=$(basename $real_nodejs_version) # Create a symbolic link for this major version if the file doesn't already exist - if [ ! -e "$node_version_path/$nodejs_version" ]; then - ln --symbolic --force --no-target-directory $node_version_path/$real_nodejs_version $node_version_path/$nodejs_version + if [ ! -e "$N_INSTALL_DIR/n/versions/node/$nodejs_version" ]; then + ln --symbolic --force --no-target-directory \ + $N_INSTALL_DIR/n/versions/node/$real_nodejs_version \ + $N_INSTALL_DIR/n/versions/node/$nodejs_version fi # Store the ID of this app and the version of node requested for it - echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version" + echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$N_INSTALL_DIR/ynh_app_version" # Store nodejs_version into the config of this app ynh_app_setting_set --key=nodejs_version --value=$nodejs_version @@ -111,17 +112,16 @@ ynh_nodejs_remove() { [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_remove" # Remove the line for this app - sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version" + sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$N_INSTALL_DIR/ynh_app_version" # If no other app uses this version of nodejs, remove it. - if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"; then - $n_install_dir/bin/n rm $nodejs_version + if ! grep --quiet "$nodejs_version" "$N_INSTALL_DIR/ynh_app_version"; then + $N_INSTALL_DIR/bin/n rm $nodejs_version fi # If no other app uses n, remove n - if [ ! -s "$n_install_dir/ynh_app_version" ]; then - ynh_safe_rm "$n_install_dir" - ynh_safe_rm "/usr/local/n" + if [ ! -s "$N_INSTALL_DIR/ynh_app_version" ]; then + ynh_safe_rm "$N_INSTALL_DIR" sed --in-place "/N_PREFIX/d" /root/.bashrc fi } diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index dae25c6fd..c33967a9f 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -1,20 +1,19 @@ #!/bin/bash -rbenv_install_dir="/opt/rbenv" -ruby_version_path="$rbenv_install_dir/versions" +readonly RBENV_INSTALL_DIR="/opt/rbenv" # RBENV_ROOT is the directory of rbenv, it needs to be loaded as a environment variable. -export RBENV_ROOT="$rbenv_install_dir" -export rbenv_root="$rbenv_install_dir" +export RBENV_ROOT="$RBENV_INSTALL_DIR" +export rbenv_root="$RBENV_INSTALL_DIR" _ynh_load_ruby_in_path_and_other_tweaks() { # Get the absolute path of this version of Ruby - local ruby_path="$ruby_version_path/$app/bin" + ruby_dir="$RBENV_INSTALL_DIR/versions/$app/bin" # Load the path of this version of ruby in $PATH - if [[ :$PATH: != *":$ruby_path"* ]]; then - PATH="$ruby_path:$PATH" + if [[ :$PATH: != *":$ruby_dir"* ]]; then + PATH="$ruby_dir:$PATH" fi # Export PATH such that it's available through sudo -E / ynh_exec_as $app @@ -26,7 +25,7 @@ _ynh_load_ruby_in_path_and_other_tweaks() { # Sets the local application-specific Ruby version pushd ${install_dir} - $rbenv_install_dir/bin/rbenv local $ruby_version + $RBENV_INSTALL_DIR/bin/rbenv local $ruby_version popd } @@ -55,7 +54,7 @@ ynh_ruby_install () { [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_install" # Load rbenv path in PATH - local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + local CLEAR_PATH="$RBENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Ruby prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -64,8 +63,8 @@ ynh_ruby_install () { test -x /usr/bin/ruby && mv /usr/bin/ruby /usr/bin/ruby_rbenv # Install or update rbenv - mkdir -p $rbenv_install_dir - rbenv="$(command -v rbenv $rbenv_install_dir/bin/rbenv | grep "$rbenv_install_dir/bin/rbenv" | head -1)" + mkdir -p $RBENV_INSTALL_DIR + rbenv="$(command -v rbenv $RBENV_INSTALL_DIR/bin/rbenv | grep "$RBENV_INSTALL_DIR/bin/rbenv" | head -1)" if [ -n "$rbenv" ]; then pushd "${rbenv%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/rbenv/rbenv.git"; then @@ -75,30 +74,30 @@ ynh_ruby_install () { else echo "Reinstalling rbenv..." cd .. - ynh_safe_rm $rbenv_install_dir - mkdir -p $rbenv_install_dir - cd $rbenv_install_dir + ynh_safe_rm $RBENV_INSTALL_DIR + mkdir -p $RBENV_INSTALL_DIR + cd $RBENV_INSTALL_DIR git init -q git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 git checkout -q -b master origin/master ynh_ruby_try_bash_extension - rbenv=$rbenv_install_dir/bin/rbenv + rbenv=$RBENV_INSTALL_DIR/bin/rbenv fi popd else echo "Installing rbenv..." - pushd $rbenv_install_dir + pushd $RBENV_INSTALL_DIR git init -q git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 git checkout -q -b master origin/master ynh_ruby_try_bash_extension - rbenv=$rbenv_install_dir/bin/rbenv + rbenv=$RBENV_INSTALL_DIR/bin/rbenv popd fi - mkdir -p "${rbenv_install_dir}/plugins" + mkdir -p "${RBENV_INSTALL_DIR}/plugins" - ruby_build="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-install rbenv-install | head -1)" + ruby_build="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-install rbenv-install | head -1)" if [ -n "$ruby_build" ]; then pushd "${ruby_build%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/rbenv/ruby-build.git"; then @@ -108,10 +107,10 @@ ynh_ruby_install () { popd else echo "Installing ruby-build..." - git clone -q https://github.com/rbenv/ruby-build.git "${rbenv_install_dir}/plugins/ruby-build" + git clone -q https://github.com/rbenv/ruby-build.git "${RBENV_INSTALL_DIR}/plugins/ruby-build" fi - rbenv_alias="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)" + rbenv_alias="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)" if [ -n "$rbenv_alias" ]; then pushd "${rbenv_alias%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/tpope/rbenv-aliases.git"; then @@ -121,10 +120,10 @@ ynh_ruby_install () { popd else echo "Installing rbenv-aliases..." - git clone -q https://github.com/tpope/rbenv-aliases.git "${rbenv_install_dir}/plugins/rbenv-aliase" + git clone -q https://github.com/tpope/rbenv-aliases.git "${RBENV_INSTALL_DIR}/plugins/rbenv-aliase" fi - rbenv_latest="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)" + rbenv_latest="$(command -v "$RBENV_INSTALL_DIR"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)" if [ -n "$rbenv_latest" ]; then pushd "${rbenv_latest%/*/*}" if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then @@ -134,14 +133,14 @@ ynh_ruby_install () { popd else echo "Installing xxenv-latest..." - git clone -q https://github.com/momo-lab/xxenv-latest.git "${rbenv_install_dir}/plugins/xxenv-latest" + git clone -q https://github.com/momo-lab/xxenv-latest.git "${RBENV_INSTALL_DIR}/plugins/xxenv-latest" fi # Enable caching - mkdir -p "${rbenv_install_dir}/cache" + mkdir -p "${RBENV_INSTALL_DIR}/cache" # Create shims directory if needed - mkdir -p "${rbenv_install_dir}/shims" + mkdir -p "${RBENV_INSTALL_DIR}/shims" # Restore /usr/local/bin in PATH PATH=$CLEAR_PATH @@ -175,8 +174,8 @@ ynh_ruby_install () { # Set environment for Ruby users echo "#rbenv -export RBENV_ROOT=$rbenv_install_dir -export PATH=\"$rbenv_install_dir/bin:$PATH\" +export RBENV_ROOT=$RBENV_INSTALL_DIR +export PATH=\"$RBENV_INSTALL_DIR/bin:$PATH\" eval \"\$(rbenv init -)\" #rbenv" > /etc/profile.d/rbenv.sh @@ -192,10 +191,11 @@ eval \"\$(rbenv init -)\" # # usage: ynh_ruby_remove ynh_ruby_remove () { - local ruby_version=$(ynh_app_setting_get --key=ruby_version) + + [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_remove" # Load rbenv path in PATH - local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + local CLEAR_PATH="$RBENV_INSTALL_DIR/bin:$PATH" # Remove /usr/local/bin in PATH in case of Ruby prior installation PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') @@ -235,7 +235,7 @@ _ynh_ruby_cleanup () { if ! echo ${required_ruby_versions} | grep -q "${installed_ruby_version}" then echo "Removing Ruby-$installed_ruby_version" - $rbenv_install_dir/bin/rbenv uninstall --force $installed_ruby_version + $RBENV_INSTALL_DIR/bin/rbenv uninstall --force $installed_ruby_version fi done @@ -244,7 +244,7 @@ _ynh_ruby_cleanup () { then # Remove rbenv environment configuration echo "Removing rbenv" - ynh_safe_rm "$rbenv_install_dir" + ynh_safe_rm "$RBENV_INSTALL_DIR" ynh_safe_rm "/etc/profile.d/rbenv.sh" fi } From 0e4495f11e3d74b97241e83345236df7c247a14e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 15:32:47 +0200 Subject: [PATCH 105/146] Update changelog for 11.2.16 --- debian/changelog | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/debian/changelog b/debian/changelog index f5059d42c..1f0c6c57a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,24 @@ +yunohost (11.2.16) stable; urgency=low + + - apps/logs: fix some information not being redacted because of the packaging v2 flows (a25033bb) + - logs: misc ad-hoc tweaks to limit the noise in log sharing (06c8fbc8) + - helpers: (1/2/2.1) add a new ynh_app_setting_set_default to replace the unecessarily complex 'if [ -z ${foo:-} ]' trick ([#1873](http://github.com/YunoHost/yunohost/pull/1873)) + - helpers2.1: drop unused 'local source' mechanism from ynh_setup_source (dd8db188) + - helpers2.1: fix positional arg parsing in ynh_psql_create_user (e5585136) + - helpers2.1: rework the fpm usage/footprint madness ([#1874](http://github.com/YunoHost/yunohost/pull/1874)) + - helpers2.1: fix ynh_config_add_logrotate when no arg is passed (3942ea12) + - helpers2.1: sudo -u$app -> sudo -u $app (d4857834) + - helpers2.1: change default timeout of ynh_systemctl to 60s instead of 300s (262453f1) + - helpers2.1: display 100 lines instead of 20 in CI context when service fails to start (9298738d) + - helpers2.1: when using ynh_systemctl to reload/start/restart a service with a wait_until and it timesout, handle it as a failure rather than keep going (b3409729) + - helpers2.1: for some reason sudo -E doesn't preserve PATH even though it's exported, so we gotta explicitly use --preserve-env=PATH (5f6df6a8) + - helpers2.1: var rename / cosmetic etc for nodejs/ruby/go version and install directories (2b1f7426) + - i18n: Translations updated for Basque, Slovak + + Thanks to all contributors <3 ! (alexAubin, Jose Riha, xabirequejo) + + -- Alexandre Aubin Sun, 23 Jun 2024 15:30:22 +0200 + yunohost (11.2.15) stable; urgency=low - apps: new experimentals "2.1" helpers ([#1855](http://github.com/YunoHost/yunohost/pull/1855)) From d8c3ff4c8ac1bd2c4bb9cd2b93a1c14f11ae7e33 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 16:35:58 +0200 Subject: [PATCH 106/146] helpers2.1: forgot to propagate the 'goenv latest' fix from helpers v1 --- helpers/helpers.v2.1.d/go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 68f5ceaad..23a423342 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -108,7 +108,7 @@ ynh_go_install () { test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go # Install the requested version of Go - local final_go_version=$(goenv latest --print "$go_version") + local final_go_version=$("$goenv_latest_dir/bin/goenv-latest" --print "$go_version") ynh_print_info "Installation of Go-$final_go_version" goenv install --quiet --skip-existing "$final_go_version" 2>&1 From 30467f8bc3f55f366b3c4ad957ccb5a4f09df5e9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 23 Jun 2024 20:04:17 +0200 Subject: [PATCH 107/146] helpers2.1: fix bad syntax in ynh_app_upstream_version --- helpers/helpers.v2.1.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index e6ad72eb1..c46c41d7c 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -460,7 +460,7 @@ ynh_read_manifest() { # # Requires YunoHost version 3.5.0 or higher. ynh_app_upstream_version() { - echo "${$YNH_APP_MANIFEST_VERSION/~ynh*/}" + echo "${YNH_APP_MANIFEST_VERSION/~ynh*/}" } # Return 0 if the "upstream" part of the version changed, or 1 otherwise (ie only the ~ynh suffix changed) From 1fb80e5d2468e8cab6f1ab08dc348e562c20b940 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 06:09:58 +0200 Subject: [PATCH 108/146] helpers2.1: drop ynh_apps helper because only a single app is using it ... --- helpers/helpers.v2.1.d/apps | 213 ------------------------------------ 1 file changed, 213 deletions(-) delete mode 100644 helpers/helpers.v2.1.d/apps diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps deleted file mode 100644 index ca76caf8d..000000000 --- a/helpers/helpers.v2.1.d/apps +++ /dev/null @@ -1,213 +0,0 @@ -#!/bin/bash - -# Install others YunoHost apps -# -# usage: ynh_install_apps --apps="appfoo?domain=domain.foo&path=/foo appbar?domain=domain.bar&path=/bar&admin=USER&language=fr&is_public=1&pass?word=pass&port=666" -# | arg: -a, --apps= - apps to install -# -# Requires YunoHost version *.*.* or higher. -ynh_install_apps() { - # ============ Argument parsing ============= - local -A args_array=([a]=apps=) - local apps - ynh_handle_getopts_args "$@" - # =========================================== - - # Split the list of apps in an array - local apps_list=($(echo $apps | tr " " "\n")) - local apps_dependencies="" - - # For each app - for one_app_and_its_args in "${apps_list[@]}" - do - # Retrieve the name of the app (part before ?) - local one_app=$(cut -d "?" -f1 <<< "$one_app_and_its_args") - [ -z "$one_app" ] && ynh_die "You didn't provided a YunoHost app to install" - - yunohost tools update apps - - # Installing or upgrading the app depending if it's installed or not - if ! yunohost app list --output-as json --quiet | jq -e --arg id $one_app '.apps[] | select(.id == $id)' >/dev/null - then - # Retrieve the arguments of the app (part after ?) - local one_argument="" - if [[ "$one_app_and_its_args" == *"?"* ]]; then - one_argument=$(cut -d "?" -f2- <<< "$one_app_and_its_args") - one_argument="--args $one_argument" - fi - - # Install the app with its arguments - yunohost app install $one_app $one_argument - else - # Upgrade the app - yunohost app upgrade $one_app - fi - - if [ ! -z "$apps_dependencies" ] - then - apps_dependencies="$apps_dependencies, $one_app" - else - apps_dependencies="$one_app" - fi - done - - ynh_app_setting_set --key=apps_dependencies --value="$apps_dependencies" -} - -# Remove other YunoHost apps -# -# Other YunoHost apps will be removed only if no other apps need them. -# -# usage: ynh_remove_apps -# -# Requires YunoHost version *.*.* or higher. -ynh_remove_apps() { - # Retrieve the apps dependencies of the app - local apps_dependencies=$(ynh_app_setting_get --key=apps_dependencies) - ynh_app_setting_delete --key=apps_dependencies - - if [ ! -z "$apps_dependencies" ] - then - # Split the list of apps dependencies in an array - local apps_dependencies_list=($(echo $apps_dependencies | tr ", " "\n")) - - # For each apps dependencies - for one_app in "${apps_dependencies_list[@]}" - do - # Retrieve the list of installed apps - local installed_apps_list=$(yunohost app list --output-as json --quiet | jq -r .apps[].id) - local required_by="" - local installed_app_required_by="" - - # For each other installed app - for one_installed_app in $installed_apps_list - do - # Retrieve the other apps dependencies - one_installed_apps_dependencies=$(ynh_app_setting_get --app=$one_installed_app --key=apps_dependencies) - if [ ! -z "$one_installed_apps_dependencies" ] - then - one_installed_apps_dependencies_list=($(echo $one_installed_apps_dependencies | tr ", " "\n")) - - # For each dependency of the other apps - for one_installed_app_dependency in "${one_installed_apps_dependencies_list[@]}" - do - if [[ $one_installed_app_dependency == $one_app ]]; then - required_by="$required_by $one_installed_app" - fi - done - fi - done - - # If $one_app is no more required - if [[ -z "$required_by" ]] - then - # Remove $one_app - ynh_print_info "Removing of $one_app" - yunohost app remove $one_app --purge - else - ynh_print_info "$one_app was not removed because it's still required by${required_by}" - fi - done - fi -} - -# Spawn a Bash shell with the app environment loaded -# -# usage: ynh_spawn_app_shell --app="app" -# | arg: -a, --app= - the app ID -# -# examples: -# ynh_spawn_app_shell --app="APP" <<< 'echo "$USER"' -# ynh_spawn_app_shell --app="APP" < /tmp/some_script.bash -# -# Requires YunoHost version 11.0.* or higher, and that the app relies on packaging v2 or higher. -# The spawned shell will have environment variables loaded and environment files sourced -# from the app's service configuration file (defaults to $app.service, overridable by the packager with `service` setting). -# If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings. -ynh_spawn_app_shell() { - # ============ Argument parsing ============= - local -A args_array=([a]=app=) - local app - ynh_handle_getopts_args "$@" - # =========================================== - - # Force Bash to be used to run this helper - if [[ ! $0 =~ \/?bash$ ]] - then - ynh_print_warn "Please use Bash as shell" - exit 1 - fi - - # Make sure the app is installed - local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id)) - if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]] - then - ynh_print_warn "$app is not in the apps list" - exit 1 - fi - - # Make sure the app has its own user - if ! id -u "$app" &>/dev/null; then - ynh_print_warn "There is no \"$app\" system user" - exit 1 - fi - - # Make sure the app has an install_dir setting - local install_dir=$(ynh_app_setting_get --key=install_dir) - if [ -z "$install_dir" ] - then - ynh_print_warn "$app has no install_dir setting (does it use packaging format >=2?)" - exit 1 - fi - - # Load the app's service name, or default to $app - local service=$(ynh_app_setting_get --key=service) - [ -z "$service" ] && service=$app; - - # Export HOME variable - export HOME=$install_dir; - - # Load the Environment variables from the app's service - local env_var=$(systemctl show $service.service -p "Environment" --value) - [ -n "$env_var" ] && export $env_var; - - # Force `php` to its intended version - # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` - local php_version=$(ynh_app_setting_get --key=php_version) - local phpflags=$(ynh_app_setting_get --key=phpflags) - if [ -n "$php_version" ] - then - eval "php() { php${php_version} ${phpflags} \"\$@\"; }" - export -f php - fi - - # Source the EnvironmentFiles from the app's service - local env_files=($(systemctl show $service.service -p "EnvironmentFiles" --value)) - if [ ${#env_files[*]} -gt 0 ] - then - # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. - set -a - for file in ${env_files[*]} - do - [[ $file = /* ]] && source $file - done - set +a - fi - - # Activate the Python environment, if it exists - if [ -f $install_dir/venv/bin/activate ] - then - # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. - set -a - source $install_dir/venv/bin/activate - set +a - fi - - # cd into the WorkingDirectory set in the service, or default to the install_dir - local env_dir=$(systemctl show $service.service -p "WorkingDirectory" --value) - [ -z $env_dir ] && env_dir=$install_dir; - cd $env_dir - - # Spawn the app shell - su -s /bin/bash $app -} From 2895d4d99bead63eeeb6c49869694e92db01600f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 06:10:34 +0200 Subject: [PATCH 109/146] helpers: Misc cleaning / reorganizing to prepare new doc --- helpers/helpers.v1.d/composer | 82 ++ helpers/helpers.v1.d/fail2ban | 4 +- helpers/helpers.v1.d/php | 81 -- helpers/helpers.v1.d/sources | 300 ++++++++ helpers/helpers.v1.d/{user => systemuser} | 54 -- helpers/helpers.v1.d/templating | 407 ++++++++++ helpers/helpers.v1.d/utils | 799 ++------------------ helpers/helpers.v2.1.d/apt | 23 +- helpers/helpers.v2.1.d/backup | 4 - helpers/helpers.v2.1.d/composer | 45 ++ helpers/helpers.v2.1.d/fail2ban | 24 +- helpers/helpers.v2.1.d/getopts | 8 +- helpers/helpers.v2.1.d/go | 37 +- helpers/helpers.v2.1.d/logging | 6 - helpers/helpers.v2.1.d/logrotate | 4 - helpers/helpers.v2.1.d/mongodb | 32 +- helpers/helpers.v2.1.d/multimedia | 12 +- helpers/helpers.v2.1.d/mysql | 14 +- helpers/helpers.v2.1.d/nginx | 6 - helpers/helpers.v2.1.d/nodejs | 15 +- helpers/helpers.v2.1.d/permission | 63 +- helpers/helpers.v2.1.d/php | 74 +- helpers/helpers.v2.1.d/postgresql | 12 +- helpers/helpers.v2.1.d/ruby | 32 +- helpers/helpers.v2.1.d/setting | 36 +- helpers/helpers.v2.1.d/sources | 261 +++++++ helpers/helpers.v2.1.d/string | 34 +- helpers/helpers.v2.1.d/systemd | 20 +- helpers/helpers.v2.1.d/{user => systemuser} | 91 +-- helpers/helpers.v2.1.d/templating | 153 ++-- helpers/helpers.v2.1.d/utils | 318 +------- 31 files changed, 1396 insertions(+), 1655 deletions(-) create mode 100644 helpers/helpers.v1.d/composer create mode 100644 helpers/helpers.v1.d/sources rename helpers/helpers.v1.d/{user => systemuser} (73%) create mode 100644 helpers/helpers.v1.d/templating create mode 100644 helpers/helpers.v2.1.d/composer create mode 100644 helpers/helpers.v2.1.d/sources rename helpers/helpers.v2.1.d/{user => systemuser} (52%) diff --git a/helpers/helpers.v1.d/composer b/helpers/helpers.v1.d/composer new file mode 100644 index 000000000..506ab8713 --- /dev/null +++ b/helpers/helpers.v1.d/composer @@ -0,0 +1,82 @@ +#!/bin/bash + +readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17 +# Declare the actual composer version to use. +# A packager willing to use another version of composer can override the variable into its _common.sh. +YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} + +# Execute a command with Composer +# +# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$install_dir] --commands="commands" +# | arg: -v, --phpversion - PHP version to use with composer +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path +# | arg: -c, --commands - Commands to execute. +# +# Requires YunoHost version 4.2 or higher. +ynh_composer_exec() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vwc + declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=) + local phpversion + local workdir + local commands + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + workdir="${workdir:-${install_dir:-$final_path}}" + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ + php${phpversion} "$workdir/composer.phar" $commands \ + -d "$workdir" --no-interaction --no-ansi 2>&1 +} + +# Install and initialize Composer in the given directory +# +# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$install_dir] [--install_args="--optimize-autoloader"] [--composerversion=composerversion] +# | arg: -v, --phpversion - PHP version to use with composer +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. +# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include +# | arg: -c, --composerversion - Composer version to install +# +# Requires YunoHost version 4.2 or higher. +ynh_install_composer() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vwac + declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) + local phpversion + local workdir + local install_args + local composerversion + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + workdir="${workdir:-$final_path}" + else + workdir="${workdir:-$install_dir}" + fi + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + install_args="${install_args:-}" + composerversion="${composerversion:-$YNH_COMPOSER_VERSION}" + + curl -sS https://getcomposer.org/installer \ + | COMPOSER_HOME="$workdir/.composer" \ + php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \ + || ynh_die --message="Unable to install Composer." + + # install dependencies + ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \ + || ynh_die --message="Unable to install core dependencies with Composer." +} diff --git a/helpers/helpers.v1.d/fail2ban b/helpers/helpers.v1.d/fail2ban index 613dcc490..db54805fc 100644 --- a/helpers/helpers.v1.d/fail2ban +++ b/helpers/helpers.v1.d/fail2ban @@ -40,9 +40,7 @@ # ignoreregex = # ``` # -# ----------------------------------------------------------------------------- -# -# Note about the "failregex" option: +# ##### Note about the "failregex" option: # # regex to match the password failure messages in the logfile. The host must be # matched by a group named "`host`". The tag "``" can be used for standard diff --git a/helpers/helpers.v1.d/php b/helpers/helpers.v1.d/php index 7fbe3f1ba..b0b573604 100644 --- a/helpers/helpers.v1.d/php +++ b/helpers/helpers.v1.d/php @@ -500,84 +500,3 @@ ynh_get_scalable_phpfpm() { fi fi } - -readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17 -# Declare the actual composer version to use. -# A packager willing to use another version of composer can override the variable into its _common.sh. -YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} - -# Execute a command with Composer -# -# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$install_dir] --commands="commands" -# | arg: -v, --phpversion - PHP version to use with composer -# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path -# | arg: -c, --commands - Commands to execute. -# -# Requires YunoHost version 4.2 or higher. -ynh_composer_exec() { - local _globalphpversion=${phpversion-:} - # Declare an array to define the options of this helper. - local legacy_args=vwc - declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=) - local phpversion - local workdir - local commands - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - workdir="${workdir:-${install_dir:-$final_path}}" - - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - phpversion="${phpversion:-$YNH_PHP_VERSION}" - else - phpversion="${phpversion:-$_globalphpversion}" - fi - - COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ - php${phpversion} "$workdir/composer.phar" $commands \ - -d "$workdir" --no-interaction --no-ansi 2>&1 -} - -# Install and initialize Composer in the given directory -# -# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$install_dir] [--install_args="--optimize-autoloader"] [--composerversion=composerversion] -# | arg: -v, --phpversion - PHP version to use with composer -# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. -# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include -# | arg: -c, --composerversion - Composer version to install -# -# Requires YunoHost version 4.2 or higher. -ynh_install_composer() { - local _globalphpversion=${phpversion-:} - # Declare an array to define the options of this helper. - local legacy_args=vwac - declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) - local phpversion - local workdir - local install_args - local composerversion - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - workdir="${workdir:-$final_path}" - else - workdir="${workdir:-$install_dir}" - fi - - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - phpversion="${phpversion:-$YNH_PHP_VERSION}" - else - phpversion="${phpversion:-$_globalphpversion}" - fi - - install_args="${install_args:-}" - composerversion="${composerversion:-$YNH_COMPOSER_VERSION}" - - curl -sS https://getcomposer.org/installer \ - | COMPOSER_HOME="$workdir/.composer" \ - php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \ - || ynh_die --message="Unable to install Composer." - - # install dependencies - ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \ - || ynh_die --message="Unable to install core dependencies with Composer." -} diff --git a/helpers/helpers.v1.d/sources b/helpers/helpers.v1.d/sources new file mode 100644 index 000000000..9bb0b1c99 --- /dev/null +++ b/helpers/helpers.v1.d/sources @@ -0,0 +1,300 @@ +#!/bin/bash + +# Download, check integrity, uncompress and patch the source from app.src +# +# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] +# | arg: -d, --dest_dir= - Directory where to setup sources +# | arg: -s, --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise +# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders) +# | arg: -r, --full_replace= - Remove previous sources before installing new sources (can be 1 or 0, default to 0) +# +# ##### New 'sources' resources +# +# (See also the resources documentation which may be more complete?) +# +# This helper will read infos from the 'sources' resources in the manifest.toml of the app +# and expect a structure like: +# +# ```toml +# [resources.sources] +# [resources.sources.main] +# url = "https://some.address.to/download/the/app/archive" +# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL +# ``` +# +# ##### Optional flags +# +# ```text +# format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract +# "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract +# "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract +# "whatever" # an arbitrary value, not really meaningful except to imply that the file won't be extracted +# +# in_subdir = true # default, there's an intermediate subdir in the archive before accessing the actual files +# false # sources are directly in the archive root +# n # (special cases) an integer representing a number of subdirs levels to get rid of +# +# extract = true # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ... +# = false # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset. +# # in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value +# +# rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical +# platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for +# ``` +# +# You may also define assets url and checksum per-architectures such as: +# ```toml +# [resources.sources] +# [resources.sources.main] +# amd64.url = "https://some.address.to/download/the/app/archive/when/amd64" +# amd64.sha256 = "0123456789abcdef" +# armhf.url = "https://some.address.to/download/the/app/archive/when/armhf" +# armhf.sha256 = "fedcba9876543210" +# ``` +# +# In which case `ynh_setup_source --dest_dir="$install_dir"` will automatically pick the appropriate source depending on the arch +# +# The helper will: +# - Download the specific URL if there is no local archive +# - Check the integrity with the specific sha256 sum +# - Uncompress the archive to `$dest_dir`. +# - If `in_subdir` is true, the first level directory of the archive will be removed. +# - If `in_subdir` is a numeric value, the N first level directories will be removed. +# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` +# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir +# +# Requires YunoHost version 2.6.4 or higher. +ynh_setup_source() { + # Declare an array to define the options of this helper. + local legacy_args=dsk + local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) + local dest_dir + local source_id + local keep + local full_replace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + keep="${keep:-}" + full_replace="${full_replace:-0}" + + if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null + then + source_id="${source_id:-main}" + local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") + if jq -re ".url" <<< "$sources_json" + then + local arch_prefix="" + else + local arch_prefix=".$YNH_ARCH" + fi + + local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" + local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" + local src_sumprg="sha256sum" + local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" + local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" + local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" + local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" + local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" + + [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" + + if [[ -z "$src_format" ]] + then + if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] + then + src_format="zip" + elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] + then + src_format="tar.gz" + elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] + then + src_format="tar.xz" + elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] + then + src_format="tar.bz2" + elif [[ -z "$src_extract" ]] + then + src_extract="false" + fi + fi + else + source_id="${source_id:-app}" + local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" + + # Load value from configuration file (see above for a small doc about this file + # format) + local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_rename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_platform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + fi + + # Default value + src_sumprg=${src_sumprg:-sha256sum} + src_in_subdir=${src_in_subdir:-true} + src_format=${src_format:-tar.gz} + src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') + src_extract=${src_extract:-true} + + if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] + then + ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" + fi + + + # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... + local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" + + # Gotta use this trick with 'dirname' because source_id may contain slashes x_x + mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) + src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" + + if [ "$src_format" = "docker" ]; then + src_platform="${src_platform:-"linux/$YNH_ARCH"}" + else + if test -e "$local_src"; then + cp $local_src $src_filename + fi + + [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" + + # If the file was prefetched but somehow doesn't match the sum, rm and redownload it + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + rm -f "$src_filename" + fi + + # Only redownload the file if it wasnt prefetched + if [ ! -e "$src_filename" ] + then + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ + || ynh_die --message="$out" + fi + + # Check the control sum + if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_size="$(du -hs ${src_filename} | cut --fields=1)" + rm -f ${src_filename} + ynh_die --message="Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." + fi + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ + if [ -n "$keep" ] && [ -e "$dest_dir" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + mkdir -p $keep_dir + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$dest_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" + cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" + fi + done + fi + + if [ "$full_replace" -eq 1 ]; then + ynh_secure_remove --file="$dest_dir" + fi + + # Extract source into the app dir + mkdir --parents "$dest_dir" + + if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then + _ynh_apply_default_permissions $dest_dir + fi + if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then + _ynh_apply_default_permissions $dest_dir + fi + + if [[ "$src_extract" == "false" ]]; then + if [[ -z "$src_rename" ]] + then + mv $src_filename $dest_dir + else + mv $src_filename $dest_dir/$src_rename + fi + elif [[ "$src_format" == "docker" ]]; then + "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1 + elif [[ "$src_format" == "zip" ]]; then + # Zip format + # Using of a temp directory, because unzip doesn't manage --strip-components + if $src_in_subdir; then + local tmp_dir=$(mktemp --directory) + unzip -quo $src_filename -d "$tmp_dir" + cp --archive $tmp_dir/*/. "$dest_dir" + ynh_secure_remove --file="$tmp_dir" + else + unzip -quo $src_filename -d "$dest_dir" + fi + ynh_secure_remove --file="$src_filename" + else + local strip="" + if [ "$src_in_subdir" != "false" ]; then + if [ "$src_in_subdir" == "true" ]; then + local sub_dirs=1 + else + local sub_dirs="$src_in_subdir" + fi + strip="--strip-components $sub_dirs" + fi + if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then + tar --extract --file=$src_filename --directory="$dest_dir" $strip + else + ynh_die --message="Archive format unrecognized." + fi + ynh_secure_remove --file="$src_filename" + fi + + # Apply patches + if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then + local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) + if (($(find $patches_folder -type f -name "${source_id}-*.patch" 2>/dev/null | wc --lines) > "0")); then + pushd "$dest_dir" + for p in $patches_folder/${source_id}-*.patch; do + echo $p + patch --strip=1 <$p || ynh_print_warn --message="Packagers /!\\ patch $p failed to apply" + done + popd + fi + fi + + # Add supplementary files + if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then + cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir" + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + if [ -n "$keep" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$keep_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" + + # We add "--no-target-directory" (short option is -T) to handle the special case + # when we "keep" a folder, but then the new setup already contains the same dir (but possibly empty) + # in which case a regular "cp" will create a copy of the directory inside the directory ... + # resulting in something like /var/www/$app/data/data instead of /var/www/$app/data + # cf https://unix.stackexchange.com/q/94831 for a more elaborate explanation on the option + cp --archive --no-target-directory "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" + fi + done + fi + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ +} diff --git a/helpers/helpers.v1.d/user b/helpers/helpers.v1.d/systemuser similarity index 73% rename from helpers/helpers.v1.d/user rename to helpers/helpers.v1.d/systemuser index e608a3308..064cae4ae 100644 --- a/helpers/helpers.v1.d/user +++ b/helpers/helpers.v1.d/systemuser @@ -1,59 +1,5 @@ #!/bin/bash -# Check if a YunoHost user exists -# -# usage: ynh_user_exists --username=username -# | arg: -u, --username= - the username to check -# | ret: 0 if the user exists, 1 otherwise. -# -# example: ynh_user_exists 'toto' || echo "User does not exist" -# -# Requires YunoHost version 2.2.4 or higher. -ynh_user_exists() { - # Declare an array to define the options of this helper. - local legacy_args=u - local -A args_array=([u]=username=) - local username - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null -} - -# Retrieve a YunoHost user information -# -# usage: ynh_user_get_info --username=username --key=key -# | arg: -u, --username= - the username to retrieve info from -# | arg: -k, --key= - the key to retrieve -# | ret: the value associate to that key -# -# example: mail=$(ynh_user_get_info --username="toto" --key=mail) -# -# Requires YunoHost version 2.2.4 or higher. -ynh_user_get_info() { - # Declare an array to define the options of this helper. - local legacy_args=uk - local -A args_array=([u]=username= [k]=key=) - local username - local key - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - yunohost user info "$username" --output-as json --quiet | jq -r ".$key" -} - -# Get the list of YunoHost users -# -# usage: ynh_user_list -# | ret: one username per line as strings -# -# example: for u in $(ynh_user_list); do ... ; done -# -# Requires YunoHost version 2.4.0 or higher. -ynh_user_list() { - yunohost user list --output-as json --quiet | jq -r ".users | keys[]" -} - # Check if a user exists on the system # # [packagingv1] diff --git a/helpers/helpers.v1.d/templating b/helpers/helpers.v1.d/templating new file mode 100644 index 000000000..76f319137 --- /dev/null +++ b/helpers/helpers.v1.d/templating @@ -0,0 +1,407 @@ +#!/bin/bash + +# Create a dedicated config file from a template +# +# usage: ynh_add_config --template="template" --destination="destination" +# | arg: -t, --template= - Template config file to use +# | arg: -d, --destination= - Destination of the config file +# | arg: -j, --jinja - Use jinja template instead of the simple `__MY_VAR__` templating format +# +# examples: +# ynh_add_config --template=".env" --destination="$install_dir/.env" # (use the template file "conf/.env" from the app's package) +# ynh_add_config --jinja --template="config.j2" --destination="$install_dir/config" # (use the template file "conf/config.j2" from the app's package) +# +# The template can be by default the name of a file in the conf directory +# of a YunoHost Package, a relative path or an absolute path. +# +# The helper will use the template `template` to generate a config file +# `destination` by replacing the following keywords with global variables +# that should be defined before calling this helper : +# ``` +# __PATH__ by $path_url +# __NAME__ by $app +# __NAMETOCHANGE__ by $app +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# ``` +# And any dynamic variables that should be defined before calling this helper like: +# ``` +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# ``` +# +# ##### When --jinja is enabled +# +# This option is meant for advanced use-cases where the "simple" templating +# mode ain't enough because you need conditional blocks or loops. +# +# For a full documentation of jinja's syntax you can refer to: +# https://jinja.palletsprojects.com/en/3.1.x/templates/ +# +# Note that in YunoHost context, all variables are from shell variables and therefore are strings +# +# ##### Keeping track of manual changes by the admin +# +# The helper will verify the checksum and backup the destination file +# if it's different before applying the new template. +# +# And it will calculate and store the destination file checksum +# into the app settings when configuration is done. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_config() { + # Declare an array to define the options of this helper. + local legacy_args=tdj + local -A args_array=([t]=template= [d]=destination= [j]=jinja) + local template + local destination + local jinja + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local template_path + jinja="${jinja:-0}" + + if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then + template_path="$YNH_APP_BASEDIR/conf/$template" + elif [ -f "$template" ]; then + template_path=$template + else + ynh_die --message="The provided template $template doesn't exist" + fi + + ynh_backup_if_checksum_is_different --file="$destination" + + # Make sure to set the permissions before we copy the file + # This is to cover a case where an attacker could have + # created a file beforehand to have control over it + # (cp won't overwrite ownership / modes by default...) + touch $destination + chmod 640 $destination + _ynh_apply_default_permissions $destination + + if [[ "$jinja" == 1 ]] + then + # This is ran in a subshell such that the "export" does not "contaminate" the main process + ( + export $(compgen -v) + j2 "$template_path" -f env -o $destination + ) + else + cp -f "$template_path" "$destination" + ynh_replace_vars --file="$destination" + fi + + ynh_store_file_checksum --file="$destination" +} + +# Replace variables in a file +# +# [internal] +# +# usage: ynh_replace_vars --file="file" +# | arg: -f, --file= - File where to replace variables +# +# The helper will replace the following keywords with global variables +# that should be defined before calling this helper : +# __PATH__ by $path_url +# __NAME__ by $app +# __NAMETOCHANGE__ by $app +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# +# And any dynamic variables that should be defined before calling this helper like: +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# +# Requires YunoHost version 4.1.0 or higher. +ynh_replace_vars() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Replace specific YunoHost variables + if test -n "${path_url:-}"; then + # path_url_slash_less is path_url, or a blank value if path_url is only '/' + local path_url_slash_less=${path_url%/} + ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file" + ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" + fi + if test -n "${app:-}"; then + ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$file" + ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file" + ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" + fi + # Legacy + if test -n "${final_path:-}"; then + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file" + ynh_replace_string --match_string="__INSTALL_DIR__" --replace_string="$final_path" --target_file="$file" + fi + # Legacy / Packaging v1 only + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2 && test -n "${YNH_PHP_VERSION:-}"; then + ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file" + fi + if test -n "${ynh_node_load_PATH:-}"; then + ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" + fi + + # Replace others variables + + # List other unique (__ __) variables in $file + local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g")) + + set +o xtrace # set +x + + # Do the replacement + local delimit=@ + for one_var in "${uniques_vars[@]}"; do + # Validate that one_var is indeed defined + # -v checks if the variable is defined, for example: + # -v FOO tests if $FOO is defined + # -v $FOO tests if ${!FOO} is defined + # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 + [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" + + # Escape delimiter in match/replace string + match_string="__${one_var^^}__" + match_string=${match_string//${delimit}/"\\${delimit}"} + replace_string="${!one_var}" + replace_string=${replace_string//\\/\\\\} + replace_string=${replace_string//${delimit}/"\\${delimit}"} + + # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs) + sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file" + done + set -o xtrace # set -x +} + +# Get a value from heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_read_var_in_file --file=PATH --key=KEY +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to get +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# This helpers match several var affectation use case in several languages +# We don't use jq or equivalent to keep comments and blank space in files +# This helpers work line by line, it is not able to work correctly +# if you have several identical keys in your files +# +# Example of line this helpers can managed correctly +# .yml +# title: YunoHost documentation +# email: 'yunohost@yunohost.org' +# .json +# "theme": "colib'ris", +# "port": 8102 +# "some_boolean": false, +# "user": null +# .ini +# some_boolean = On +# action = "Clear" +# port = 20 +# .php +# $user= +# user => 20 +# .py +# USER = 8102 +# user = 'https://donate.local' +# CUSTOM['user'] = 'YunoHost' +# +# Requires YunoHost version 4.3 or higher. +ynh_read_var_in_file() { + # Declare an array to define the options of this helper. + local legacy_args=fka + local -A args_array=([f]=file= [k]=key= [a]=after=) + local file + local key + local after + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + after="${after:-}" + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local line_number=1 + if [[ -n "$after" ]]; then + line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + echo YNH_NULL + return 0 + fi + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + + local first_char="${expression:0:1}" + if [[ "$first_char" == '"' ]]; then + echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + elif [[ "$first_char" == "'" ]]; then + echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + else + echo "$expression" + fi + set -o xtrace # set -x +} + +# Set a value into heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to set +# | arg: -v, --value= - the value to set +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# Requires YunoHost version 4.3 or higher. +ynh_write_var_in_file() { + # Declare an array to define the options of this helper. + local legacy_args=fkva + local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) + local file + local key + local value + local after + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + after="${after:-}" + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local after_line_number=1 + if [[ -n "$after" ]]; then + after_line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$after_line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$after_line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + return 1 + fi + local value_line_number="$(tail +$after_line_number ${file} | grep -m1 -n -i -P $var_part'\K.*$' | cut -d: -f1)" + value_line_number=$((after_line_number + value_line_number)) + local range="${after_line_number},${value_line_number} " + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + endline=${expression_with_comment#"$expression"} + endline="$(echo "$endline" | sed 's/\\/\\\\/g')" + value="$(echo "$value" | sed 's/\\/\\\\/g')" + value=${value//&/"\&"} + local first_char="${expression:0:1}" + delimiter=$'\001' + if [[ "$first_char" == '"' ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # So we need \\\\ to go through 2 sed + value="$(echo "$value" | sed 's/"/\\\\"/g')" + sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file} + elif [[ "$first_char" == "'" ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # However double quotes implies to double \\ to + # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str + value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" + sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file} + else + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]]; then + value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' + fi + if [[ "$ext" =~ ^yaml|yml$ ]]; then + value=" $value" + fi + sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file} + fi + set -o xtrace # set -x +} + +# Render templates with Jinja2 +# +# [internal] +# +# Attention : Variables should be exported before calling this helper to be +# accessible inside templates. +# +# usage: ynh_render_template some_template output_path +# | arg: some_template - Template file to be rendered +# | arg: output_path - The path where the output will be redirected to +ynh_render_template() { + local template_path=$1 + local output_path=$2 + mkdir -p "$(dirname $output_path)" + # Taken from https://stackoverflow.com/a/35009576 + python3 -c 'import os, sys, jinja2; sys.stdout.write( + jinja2.Template(sys.stdin.read() + ).render(os.environ));' <$template_path >$output_path +} diff --git a/helpers/helpers.v1.d/utils b/helpers/helpers.v1.d/utils index a29b61a32..33cffc975 100644 --- a/helpers/helpers.v1.d/utils +++ b/helpers/helpers.v1.d/utils @@ -72,322 +72,6 @@ then ynh_abort_if_errors fi -# Download, check integrity, uncompress and patch the source from app.src -# -# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] -# | arg: -d, --dest_dir= - Directory where to setup sources -# | arg: -s, --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise -# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders) -# | arg: -r, --full_replace= - Remove previous sources before installing new sources (can be 1 or 0, default to 0) -# -# #### New 'sources' resources -# -# (See also the resources documentation which may be more complete?) -# -# This helper will read infos from the 'sources' resources in the manifest.toml of the app -# and expect a structure like: -# -# ```toml -# [resources.sources] -# [resources.sources.main] -# url = "https://some.address.to/download/the/app/archive" -# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL -# ``` -# -# ##### Optional flags -# -# ```text -# format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract -# "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract -# "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract -# "whatever" # an arbitrary value, not really meaningful except to imply that the file won't be extracted -# -# in_subdir = true # default, there's an intermediate subdir in the archive before accessing the actual files -# false # sources are directly in the archive root -# n # (special cases) an integer representing a number of subdirs levels to get rid of -# -# extract = true # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ... -# = false # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset. -# # in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value -# -# rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical -# platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for -# ``` -# -# You may also define assets url and checksum per-architectures such as: -# ```toml -# [resources.sources] -# [resources.sources.main] -# amd64.url = "https://some.address.to/download/the/app/archive/when/amd64" -# amd64.sha256 = "0123456789abcdef" -# armhf.url = "https://some.address.to/download/the/app/archive/when/armhf" -# armhf.sha256 = "fedcba9876543210" -# ``` -# -# In which case ynh_setup_source --dest_dir="$install_dir" will automatically pick the appropriate source depending on the arch -# -# -# -# #### Legacy format '.src' -# -# This helper will read `conf/${source_id}.src`, download and install the sources. -# -# The src file need to contains: -# ``` -# SOURCE_URL=Address to download the app archive -# SOURCE_SUM=Sha256 sum -# SOURCE_FORMAT=tar.gz -# SOURCE_IN_SUBDIR=false -# SOURCE_FILENAME=example.tar.gz -# SOURCE_EXTRACT=(true|false) -# SOURCE_PLATFORM=linux/arm64/v8 -# ``` -# -# The helper will: -# - Download the specific URL if there is no local archive -# - Check the integrity with the specific sha256 sum -# - Uncompress the archive to `$dest_dir`. -# - If `in_subdir` is true, the first level directory of the archive will be removed. -# - If `in_subdir` is a numeric value, the N first level directories will be removed. -# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` -# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir -# -# Requires YunoHost version 2.6.4 or higher. -ynh_setup_source() { - # Declare an array to define the options of this helper. - local legacy_args=dsk - local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) - local dest_dir - local source_id - local keep - local full_replace - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - keep="${keep:-}" - full_replace="${full_replace:-0}" - - if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null - then - source_id="${source_id:-main}" - local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") - if jq -re ".url" <<< "$sources_json" - then - local arch_prefix="" - else - local arch_prefix=".$YNH_ARCH" - fi - - local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" - local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" - local src_sumprg="sha256sum" - local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" - local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" - local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" - local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" - local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" - - [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" - [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" - - if [[ -z "$src_format" ]] - then - if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] - then - src_format="zip" - elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] - then - src_format="tar.gz" - elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] - then - src_format="tar.xz" - elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] - then - src_format="tar.bz2" - elif [[ -z "$src_extract" ]] - then - src_extract="false" - fi - fi - else - source_id="${source_id:-app}" - local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" - - # Load value from configuration file (see above for a small doc about this file - # format) - local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_rename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) - local src_platform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) - fi - - # Default value - src_sumprg=${src_sumprg:-sha256sum} - src_in_subdir=${src_in_subdir:-true} - src_format=${src_format:-tar.gz} - src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') - src_extract=${src_extract:-true} - - if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] - then - ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" - fi - - - # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... - local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" - - # Gotta use this trick with 'dirname' because source_id may contain slashes x_x - mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) - src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" - - if [ "$src_format" = "docker" ]; then - src_platform="${src_platform:-"linux/$YNH_ARCH"}" - else - if test -e "$local_src"; then - cp $local_src $src_filename - fi - - [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" - - # If the file was prefetched but somehow doesn't match the sum, rm and redownload it - if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status - then - rm -f "$src_filename" - fi - - # Only redownload the file if it wasnt prefetched - if [ ! -e "$src_filename" ] - then - # NB. we have to declare the var as local first, - # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work - # because local always return 0 ... - local out - # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) - out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ - || ynh_die --message="$out" - fi - - # Check the control sum - if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status - then - local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" - local actual_size="$(du -hs ${src_filename} | cut --fields=1)" - rm -f ${src_filename} - ynh_die --message="Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." - fi - fi - - # Keep files to be backup/restored at the end of the helper - # Assuming $dest_dir already exists - rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ - if [ -n "$keep" ] && [ -e "$dest_dir" ]; then - local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} - mkdir -p $keep_dir - local stuff_to_keep - for stuff_to_keep in $keep; do - if [ -e "$dest_dir/$stuff_to_keep" ]; then - mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" - cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" - fi - done - fi - - if [ "$full_replace" -eq 1 ]; then - ynh_secure_remove --file="$dest_dir" - fi - - # Extract source into the app dir - mkdir --parents "$dest_dir" - - if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then - _ynh_apply_default_permissions $dest_dir - fi - if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then - _ynh_apply_default_permissions $dest_dir - fi - - if [[ "$src_extract" == "false" ]]; then - if [[ -z "$src_rename" ]] - then - mv $src_filename $dest_dir - else - mv $src_filename $dest_dir/$src_rename - fi - elif [[ "$src_format" == "docker" ]]; then - "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1 - elif [[ "$src_format" == "zip" ]]; then - # Zip format - # Using of a temp directory, because unzip doesn't manage --strip-components - if $src_in_subdir; then - local tmp_dir=$(mktemp --directory) - unzip -quo $src_filename -d "$tmp_dir" - cp --archive $tmp_dir/*/. "$dest_dir" - ynh_secure_remove --file="$tmp_dir" - else - unzip -quo $src_filename -d "$dest_dir" - fi - ynh_secure_remove --file="$src_filename" - else - local strip="" - if [ "$src_in_subdir" != "false" ]; then - if [ "$src_in_subdir" == "true" ]; then - local sub_dirs=1 - else - local sub_dirs="$src_in_subdir" - fi - strip="--strip-components $sub_dirs" - fi - if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then - tar --extract --file=$src_filename --directory="$dest_dir" $strip - else - ynh_die --message="Archive format unrecognized." - fi - ynh_secure_remove --file="$src_filename" - fi - - # Apply patches - if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then - local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) - if (($(find $patches_folder -type f -name "${source_id}-*.patch" 2>/dev/null | wc --lines) > "0")); then - pushd "$dest_dir" - for p in $patches_folder/${source_id}-*.patch; do - echo $p - patch --strip=1 <$p || ynh_print_warn --message="Packagers /!\\ patch $p failed to apply" - done - popd - fi - fi - - # Add supplementary files - if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then - cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir" - fi - - # Keep files to be backup/restored at the end of the helper - # Assuming $dest_dir already exists - if [ -n "$keep" ]; then - local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} - local stuff_to_keep - for stuff_to_keep in $keep; do - if [ -e "$keep_dir/$stuff_to_keep" ]; then - mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" - - # We add "--no-target-directory" (short option is -T) to handle the special case - # when we "keep" a folder, but then the new setup already contains the same dir (but possibly empty) - # in which case a regular "cp" will create a copy of the directory inside the directory ... - # resulting in something like /var/www/$app/data/data instead of /var/www/$app/data - # cf https://unix.stackexchange.com/q/94831 for a more elaborate explanation on the option - cp --archive --no-target-directory "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" - fi - done - fi - rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ -} - # Curl abstraction to help with POST requests to local pages (such as installation forms) # # usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ... @@ -447,435 +131,6 @@ ynh_local_curl() { fi } -# Create a dedicated config file from a template -# -# usage: ynh_add_config --template="template" --destination="destination" -# | arg: -t, --template= - Template config file to use -# | arg: -d, --destination= - Destination of the config file -# | arg: -j, --jinja - Use jinja template instead of legacy __MY_VAR__ -# -# examples: -# ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" -# ynh_add_config --jinja --template="config.j2" --destination="$install_dir/config" use the template file "../conf/config.j2" -# ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" -# -## -## How it works in "legacy" mode -## -# The template can be by default the name of a file in the conf directory -# of a YunoHost Package, a relative path or an absolute path. -# -# The helper will use the template `template` to generate a config file -# `destination` by replacing the following keywords with global variables -# that should be defined before calling this helper : -# ``` -# __PATH__ by $path_url -# __NAME__ by $app -# __NAMETOCHANGE__ by $app -# __USER__ by $app -# __FINALPATH__ by $final_path -# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) -# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH -# ``` -# And any dynamic variables that should be defined before calling this helper like: -# ``` -# __DOMAIN__ by $domain -# __APP__ by $app -# __VAR_1__ by $var_1 -# __VAR_2__ by $var_2 -# ``` -# -## -## When --jinja is enabled -## -# For a full documentation of the template you can refer to: https://jinja.palletsprojects.com/en/3.1.x/templates/ -# In Yunohost context there are no really some specificity except that all variable passed are of type string. -# So here are some example of recommended usage: -# -# If you need a conditional block -# -# {% if should_my_block_be_shown == 'true' %} -# ... -# {% endif %} -# -# or -# -# {% if should_my_block_be_shown == '1' %} -# ... -# {% endif %} -# -# If you need to iterate with loop: -# -# {% for yolo in var_with_multiline_value.splitlines() %} -# ... -# {% endfor %} -# -# or -# -# {% for jail in my_var_with_coma.split(',') %} -# ... -# {% endfor %} -# -# The helper will verify the checksum and backup the destination file -# if it's different before applying the new template. -# -# And it will calculate and store the destination file checksum -# into the app settings when configuration is done. -# -# Requires YunoHost version 4.1.0 or higher. -ynh_add_config() { - # Declare an array to define the options of this helper. - local legacy_args=tdj - local -A args_array=([t]=template= [d]=destination= [j]=jinja) - local template - local destination - local jinja - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - local template_path - jinja="${jinja:-0}" - - if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then - template_path="$YNH_APP_BASEDIR/conf/$template" - elif [ -f "$template" ]; then - template_path=$template - else - ynh_die --message="The provided template $template doesn't exist" - fi - - ynh_backup_if_checksum_is_different --file="$destination" - - # Make sure to set the permissions before we copy the file - # This is to cover a case where an attacker could have - # created a file beforehand to have control over it - # (cp won't overwrite ownership / modes by default...) - touch $destination - chmod 640 $destination - _ynh_apply_default_permissions $destination - - if [[ "$jinja" == 1 ]] - then - # This is ran in a subshell such that the "export" does not "contaminate" the main process - ( - export $(compgen -v) - j2 "$template_path" -f env -o $destination - ) - else - cp -f "$template_path" "$destination" - ynh_replace_vars --file="$destination" - fi - - ynh_store_file_checksum --file="$destination" -} - -# Replace variables in a file -# -# [internal] -# -# usage: ynh_replace_vars --file="file" -# | arg: -f, --file= - File where to replace variables -# -# The helper will replace the following keywords with global variables -# that should be defined before calling this helper : -# __PATH__ by $path_url -# __NAME__ by $app -# __NAMETOCHANGE__ by $app -# __USER__ by $app -# __FINALPATH__ by $final_path -# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) -# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH -# -# And any dynamic variables that should be defined before calling this helper like: -# __DOMAIN__ by $domain -# __APP__ by $app -# __VAR_1__ by $var_1 -# __VAR_2__ by $var_2 -# -# Requires YunoHost version 4.1.0 or higher. -ynh_replace_vars() { - # Declare an array to define the options of this helper. - local legacy_args=f - local -A args_array=([f]=file=) - local file - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - # Replace specific YunoHost variables - if test -n "${path_url:-}"; then - # path_url_slash_less is path_url, or a blank value if path_url is only '/' - local path_url_slash_less=${path_url%/} - ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file" - ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" - fi - if test -n "${app:-}"; then - ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$file" - ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file" - ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" - fi - # Legacy - if test -n "${final_path:-}"; then - ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file" - ynh_replace_string --match_string="__INSTALL_DIR__" --replace_string="$final_path" --target_file="$file" - fi - # Legacy / Packaging v1 only - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2 && test -n "${YNH_PHP_VERSION:-}"; then - ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file" - fi - if test -n "${ynh_node_load_PATH:-}"; then - ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" - fi - - # Replace others variables - - # List other unique (__ __) variables in $file - local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g")) - - set +o xtrace # set +x - - # Do the replacement - local delimit=@ - for one_var in "${uniques_vars[@]}"; do - # Validate that one_var is indeed defined - # -v checks if the variable is defined, for example: - # -v FOO tests if $FOO is defined - # -v $FOO tests if ${!FOO} is defined - # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 - [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" - - # Escape delimiter in match/replace string - match_string="__${one_var^^}__" - match_string=${match_string//${delimit}/"\\${delimit}"} - replace_string="${!one_var}" - replace_string=${replace_string//\\/\\\\} - replace_string=${replace_string//${delimit}/"\\${delimit}"} - - # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs) - sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file" - done - set -o xtrace # set -x -} - -# Get a value from heterogeneous file (yaml, json, php, python...) -# -# usage: ynh_read_var_in_file --file=PATH --key=KEY -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to get -# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) -# -# This helpers match several var affectation use case in several languages -# We don't use jq or equivalent to keep comments and blank space in files -# This helpers work line by line, it is not able to work correctly -# if you have several identical keys in your files -# -# Example of line this helpers can managed correctly -# .yml -# title: YunoHost documentation -# email: 'yunohost@yunohost.org' -# .json -# "theme": "colib'ris", -# "port": 8102 -# "some_boolean": false, -# "user": null -# .ini -# some_boolean = On -# action = "Clear" -# port = 20 -# .php -# $user= -# user => 20 -# .py -# USER = 8102 -# user = 'https://donate.local' -# CUSTOM['user'] = 'YunoHost' -# -# Requires YunoHost version 4.3 or higher. -ynh_read_var_in_file() { - # Declare an array to define the options of this helper. - local legacy_args=fka - local -A args_array=([f]=file= [k]=key= [a]=after=) - local file - local key - local after - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - after="${after:-}" - - [[ -f $file ]] || ynh_die --message="File $file does not exists" - - set +o xtrace # set +x - - # Get the line number after which we search for the variable - local line_number=1 - if [[ -n "$after" ]]; then - line_number=$(grep -m1 -n $after $file | cut -d: -f1) - if [[ -z "$line_number" ]]; then - set -o xtrace # set -x - return 1 - fi - fi - - local filename="$(basename -- "$file")" - local ext="${filename##*.}" - local endline=',;' - local assign="=>|:|=" - local comments="#" - local string="\"'" - if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then - endline='#' - fi - if [[ "$ext" =~ ^ini|env$ ]]; then - comments="[;#]" - fi - if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then - comments="//" - fi - local list='\[\s*['$string']?\w+['$string']?\]' - local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' - var_part+="[$string]?${key}[$string]?" - var_part+='\s*\]?\s*' - var_part+="($assign)" - var_part+='\s*' - - # Extract the part after assignation sign - local expression_with_comment="$((tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" - if [[ "$expression_with_comment" == "YNH_NULL" ]]; then - set -o xtrace # set -x - echo YNH_NULL - return 0 - fi - - # Remove comments if needed - local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" - - local first_char="${expression:0:1}" - if [[ "$first_char" == '"' ]]; then - echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' - elif [[ "$first_char" == "'" ]]; then - echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" - else - echo "$expression" - fi - set -o xtrace # set -x -} - -# Set a value into heterogeneous file (yaml, json, php, python...) -# -# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to set -# | arg: -v, --value= - the value to set -# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) -# -# Requires YunoHost version 4.3 or higher. -ynh_write_var_in_file() { - # Declare an array to define the options of this helper. - local legacy_args=fkva - local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) - local file - local key - local value - local after - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - after="${after:-}" - - [[ -f $file ]] || ynh_die --message="File $file does not exists" - - set +o xtrace # set +x - - # Get the line number after which we search for the variable - local after_line_number=1 - if [[ -n "$after" ]]; then - after_line_number=$(grep -m1 -n $after $file | cut -d: -f1) - if [[ -z "$after_line_number" ]]; then - set -o xtrace # set -x - return 1 - fi - fi - - local filename="$(basename -- "$file")" - local ext="${filename##*.}" - local endline=',;' - local assign="=>|:|=" - local comments="#" - local string="\"'" - if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then - endline='#' - fi - if [[ "$ext" =~ ^ini|env$ ]]; then - comments="[;#]" - fi - if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then - comments="//" - fi - local list='\[\s*['$string']?\w+['$string']?\]' - local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' - var_part+="[$string]?${key}[$string]?" - var_part+='\s*\]?\s*' - var_part+="($assign)" - var_part+='\s*' - - # Extract the part after assignation sign - local expression_with_comment="$((tail +$after_line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" - if [[ "$expression_with_comment" == "YNH_NULL" ]]; then - set -o xtrace # set -x - return 1 - fi - local value_line_number="$(tail +$after_line_number ${file} | grep -m1 -n -i -P $var_part'\K.*$' | cut -d: -f1)" - value_line_number=$((after_line_number + value_line_number)) - local range="${after_line_number},${value_line_number} " - - # Remove comments if needed - local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" - endline=${expression_with_comment#"$expression"} - endline="$(echo "$endline" | sed 's/\\/\\\\/g')" - value="$(echo "$value" | sed 's/\\/\\\\/g')" - value=${value//&/"\&"} - local first_char="${expression:0:1}" - delimiter=$'\001' - if [[ "$first_char" == '"' ]]; then - # \ and sed is quite complex you need 2 \\ to get one in a sed - # So we need \\\\ to go through 2 sed - value="$(echo "$value" | sed 's/"/\\\\"/g')" - sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file} - elif [[ "$first_char" == "'" ]]; then - # \ and sed is quite complex you need 2 \\ to get one in a sed - # However double quotes implies to double \\ to - # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str - value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" - sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file} - else - if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]]; then - value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' - fi - if [[ "$ext" =~ ^yaml|yml$ ]]; then - value=" $value" - fi - sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file} - fi - set -o xtrace # set -x -} - -# Render templates with Jinja2 -# -# [internal] -# -# Attention : Variables should be exported before calling this helper to be -# accessible inside templates. -# -# usage: ynh_render_template some_template output_path -# | arg: some_template - Template file to be rendered -# | arg: output_path - The path where the output will be redirected to -ynh_render_template() { - local template_path=$1 - local output_path=$2 - mkdir -p "$(dirname $output_path)" - # Taken from https://stackoverflow.com/a/35009576 - python3 -c 'import os, sys, jinja2; sys.stdout.write( - jinja2.Template(sys.stdin.read() - ).render(os.environ));' <$template_path >$output_path -} - # Fetch the Debian release codename # # [packagingv1] @@ -1146,3 +401,57 @@ int_to_bool() { toml_to_json() { python3 -c 'import toml, json, sys; print(json.dumps(toml.load(sys.stdin)))' } + +# Check if a YunoHost user exists +# +# usage: ynh_user_exists --username=username +# | arg: -u, --username= - the username to check +# | ret: 0 if the user exists, 1 otherwise. +# +# example: ynh_user_exists 'toto' || echo "User does not exist" +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null +} + +# Retrieve a YunoHost user information +# +# usage: ynh_user_get_info --username=username --key=key +# | arg: -u, --username= - the username to retrieve info from +# | arg: -k, --key= - the key to retrieve +# | ret: the value associate to that key +# +# example: mail=$(ynh_user_get_info --username="toto" --key=mail) +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_get_info() { + # Declare an array to define the options of this helper. + local legacy_args=uk + local -A args_array=([u]=username= [k]=key=) + local username + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user info "$username" --output-as json --quiet | jq -r ".$key" +} + +# Get the list of YunoHost users +# +# usage: ynh_user_list +# | ret: one username per line as strings +# +# example: for u in $(ynh_user_list); do ... ; done +# +# Requires YunoHost version 2.4.0 or higher. +ynh_user_list() { + yunohost user list --output-as json --quiet | jq -r ".users | keys[]" +} diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 296e17b95..00a48d64d 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -4,17 +4,12 @@ YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true" # Define and install dependencies with a equivs control file # -# [packagingv1] -# -# This helper can/should only be called once per app -# # example : ynh_install_app_dependencies dep1 dep2 "dep3|dep4|dep5" # # usage: ynh_install_app_dependencies dep [dep [...]] # | arg: dep - the package name to install in dependence. # | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc). # -# Requires YunoHost version 2.6.4 or higher. ynh_apt_install_dependencies() { # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) @@ -158,13 +153,9 @@ EOF # Remove fake package and its dependencies # -# [packagingv1] -# # Dependencies will removed only if no other package need them. # # usage: ynh_apt_remove_dependencies -# -# Requires YunoHost version 2.6.4 or higher. ynh_apt_remove_dependencies() { local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps @@ -192,14 +183,11 @@ ynh_apt_remove_dependencies() { # Install packages from an extra repository properly. # -# [packagingv1] -# # usage: ynh_apt_install_dependencies_from_extra_repository --repo="repo" --package="dep1 dep2" --key=key_url -# | arg: -r, --repo= - Complete url of the extra repository. -# | arg: -p, --package= - The packages to install from this extra repository -# | arg: -k, --key= - url to get the public key. +# | arg: --repo= - Complete url of the extra repository. +# | arg: --package= - The packages to install from this extra repository +# | arg: --key= - url to get the public key. # -# Requires YunoHost version 3.8.1 or higher. ynh_apt_install_dependencies_from_extra_repository() { # ============ Argument parsing ============= local -A args_array=([r]=repo= [p]=package= [k]=key=) @@ -262,10 +250,9 @@ EOF ynh_apt_update } - -####################### +# ##################### # Internal misc utils # -####################### +# ##################### # Check if apt is free to use, or wait, until timeout. _ynh_wait_dpkg_free() { diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index c310cc0b8..0668d3e17 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -128,8 +128,6 @@ with open(sys.argv[1], 'r') as backup_file: # `/etc/nginx/conf.d/$domain.d/$app.conf` # otheriwse, search for a match in the csv (eg: conf/nginx.conf) and restore it into # `/etc/nginx/conf.d/$domain.d/$app.conf` -# -# Requires YunoHost version 2.6.4 or higher. ynh_restore() { target="$1" @@ -186,8 +184,6 @@ ynh_restore() { # Restore all files that were previously backuped in an app backup script # # usage: ynh_restore_everything -# -# Requires YunoHost version 2.6.4 or higher. ynh_restore_everything() { # Deduce the relative path of $YNH_CWD local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}" diff --git a/helpers/helpers.v2.1.d/composer b/helpers/helpers.v2.1.d/composer new file mode 100644 index 000000000..b9608f693 --- /dev/null +++ b/helpers/helpers.v2.1.d/composer @@ -0,0 +1,45 @@ +#!/bin/bash + +# Install and initialize Composer in the given directory +# +# The installed version is defined by `$composer_version` which should be defined +# as global prior to calling this helper. +# +# Will use `$install_dir` as workdir unless `$composer_workdir` exists (but that shouldnt be necessary) +# +# usage: ynh_composer_install +ynh_composer_install() { + local workdir="${composer_workdir:-$install_dir}" + + [[ -n "${composer_version}" ]] || ynh_die "\$composer_version should be defined before calling ynh_composer_install. (In the past, this was called \$YNH_COMPOSER_VERSION)" + + [[ ! -e "$workdir/composer.phar" ]] || ynh_safe_rm $workdir/composer.phar + + local composer_url="https://getcomposer.org/download/$composer_version/composer.phar" + + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir/composer.phar $composer_url 2>&1) \ + || ynh_die "$out" +} + +# Execute a command with Composer +# +# Will use `$install_dir` as workdir unless `$composer_workdir` exists (but that shouldnt be necessary) +# +# You may also define `composer_user=root` prior to call this helper if you +# absolutely need composer to run as root, but this is discouraged... +# +# usage: ynh_composer_exec commands +ynh_composer_exec() { + local workdir="${composer_workdir:-$install_dir}" + + COMPOSER_HOME="$workdir/.composer" \ + COMPOSER_MEMORY_LIMIT=-1 \ + sudo -E -u "${composer_user:-$app}" \ + php${php_version} "$workdir/composer.phar" $@ \ + -d "$workdir" --no-interaction --no-ansi 2>&1 +} diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index a382d5dce..6ca379074 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -2,18 +2,16 @@ # Create a dedicated fail2ban config (jail and filter conf files) # -# usage 1: ynh_config_add_fail2ban --logpath=log_file --failregex=filter -# | arg: -l, --logpath= - Log file to be checked by fail2ban -# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# usage: ynh_config_add_fail2ban --logpath=log_file --failregex=filter +# | arg: --logpath= - Log file to be checked by fail2ban +# | arg: --failregex= - Failregex to be looked for by fail2ban # -# usage 2: ynh_config_add_fail2ban -# | arg: -t, --use_template - Use this helper in template mode +# If --logpath / --failregex are provided, the helper will generate the appropriate conf using these. # -# This will use a template in `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf` -# See the documentation of `ynh_config_add` for a description of the template -# format and how placeholders are replaced with actual variables. +# Otherwise, it will assume that the app provided templates, namely +# `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf` # -# Generally your template will look like that by example (for synapse): +# They will typically look like (for example here for synapse): # ``` # f2b_jail.conf: # [__APP__] @@ -38,9 +36,7 @@ # ignoreregex = # ``` # -# ----------------------------------------------------------------------------- -# -# Note about the "failregex" option: +# ##### Regarding the the `failregex` option: # # regex to match the password failure messages in the logfile. The host must be # matched by a group named "`host`". The tag "``" can be used for standard @@ -53,8 +49,6 @@ # ``` # fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf # ``` -# -# Requires YunoHost version 4.1.0 or higher. ynh_config_add_fail2ban() { # ============ Argument parsing ============= local -A args_array=([l]=logpath= [r]=failregex=) @@ -117,8 +111,6 @@ ignoreregex = # Remove the dedicated fail2ban config (jail and filter conf files) # # usage: ynh_config_remove_fail2ban -# -# Requires YunoHost version 3.5.0 or higher. ynh_config_remove_fail2ban() { ynh_safe_rm "/etc/fail2ban/jail.d/$app.conf" ynh_safe_rm "/etc/fail2ban/filter.d/$app.conf" diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index 381c74256..d8373203d 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -40,11 +40,6 @@ # If there's many values for an option, -f /final /path, the value will be separated by a ';' $finalpath=/final;/path # For an option without value, like --user in the example, the helper can be called only with --user or -u. $user will then get the value 1. # -# To keep a retrocompatibility, a package can still call a helper, using getopts, with positional arguments. -# The "legacy mode" will manage the positional arguments and fill the variable in the same order than they are given in $args_array. -# e.g. for `my_helper "val1" val2`, arg1 will be filled with val1, and arg2 with val2. -# -# Requires YunoHost version 3.2.2 or higher. ynh_handle_getopts_args() { # Trick to only re-enable debugging if it was set before local xtrace_enable=$(set +o | grep xtrace) @@ -54,6 +49,9 @@ ynh_handle_getopts_args() { if [ $# -eq 0 ]; then eval "$xtrace_enable" return + # Validate that the first char is - because it should be something like --option=value or -o ... + elif [[ "${1:0:1}" != "-" ]] + ynh_die "It looks like you called the helper using positional arguments instead of keyword arguments ?" fi # Store arguments in an array to keep each argument separated diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index 23a423342..c0b8e9022 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -1,13 +1,5 @@ #!/bin/bash -ynh_go_try_bash_extension() { - if [ -x src/configure ]; then - src/configure && make -C src || { - ynh_print_info "Optional bash extension failed to build, but things will still work normally." - } - fi -} - readonly GOENV_INSTALL_DIR="/opt/goenv" # goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable. export GOENV_ROOT="$GOENV_INSTALL_DIR" @@ -37,19 +29,16 @@ _ynh_load_go_in_path_and_other_tweaks() { # Install a specific version of Go using goenv # -# The installed version is defined by $nodejs_version which should be defined as global prior to calling this helper -# -# This helper creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv -# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) -# -# Don't forget to execute go-dependent command in a login environment -# (e.g. sudo --login option) -# When not possible (e.g. in systemd service definition), please use direct path -# to goenv shims (e.g. $goenv_ROOT/shims/bundle) +# The installed version is defined by `$go_version` which should be defined as global prior to calling this helper # # usage: ynh_go_install # -# Requires YunoHost version 3.2.2 or higher. +# The helper adds the appropriate, specific version of go to the `$PATH` variable (which +# is preserved when calling `ynh_exec_as_app`). Also defines: +# - `$path_with_go` (the value of the modified `$PATH`, but you dont really need it?) +# - `$go_dir` (the directory containing the specific go version) +# +# This helper also creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv ynh_go_install () { [[ -n "${go_version:-}" ]] || ynh_die "\$go_version should be defined prior to calling ynh_go_install" @@ -76,7 +65,7 @@ ynh_go_install () { git fetch -q --tags --prune origin local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") git checkout -q "$git_latest_tag" - ynh_go_try_bash_extension + _ynh_go_try_bash_extension goenv=$GOENV_INSTALL_DIR/bin/goenv popd @@ -155,6 +144,8 @@ ynh_go_remove () { # Remove no more needed versions of Go used by the app. # +# [internal] +# # This helper will check what Go version are no more required, # and uninstall them # If no app uses Go, goenv will be also removed. @@ -194,3 +185,11 @@ _ynh_go_cleanup () { ynh_safe_rm "/etc/profile.d/goenv.sh" fi } + +_ynh_go_try_bash_extension() { + if [ -x src/configure ]; then + src/configure && make -C src || { + ynh_print_info "Optional bash extension failed to build, but things will still work normally." + } + fi +} diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging index d10330581..4649cf670 100644 --- a/helpers/helpers.v2.1.d/logging +++ b/helpers/helpers.v2.1.d/logging @@ -38,8 +38,6 @@ ynh_hide_warnings() { # | arg: command - command to execute # # Note that you should NOT quote the command but only prefix it with ynh_exec_and_print_stderr_only_if_error -# -# Requires YunoHost version 11.2 or higher. ynh_exec_and_print_stderr_only_if_error() { logfile="$(mktemp)" rc=0 @@ -56,8 +54,6 @@ ynh_exec_and_print_stderr_only_if_error() { # (to be used by special hooks like app config panel and core diagnosis) # # usage: ynh_return somedata -# -# Requires YunoHost version 3.6.0 or higher. ynh_return() { echo "$1" >>"$YNH_STDRETURN" } @@ -76,8 +72,6 @@ progress_string0="...................." # Print a progress bar showing the progression of an app script # # usage: ynh_script_progression "Some message" -# -# Requires YunoHost version 3.5.0 or higher. ynh_script_progression() { set +o xtrace # set +x diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index cd10e29d6..07e62261f 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -10,8 +10,6 @@ FIRST_CALL_TO_LOGROTATE="true" # # The configuration is autogenerated by YunoHost # (ie it doesnt come from a specific app template like nginx or systemd conf) -# -# Requires YunoHost version 2.6.4 or higher. ynh_config_add_logrotate() { local logfile="${1:-}" @@ -68,8 +66,6 @@ EOF # Remove the app's logrotate config. # # usage: ynh_remove_logrotate -# -# Requires YunoHost version 2.6.4 or higher. ynh_config_remove_logrotate() { if [ -e "/etc/logrotate.d/$app" ]; then rm "/etc/logrotate.d/$app" diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index 370b88d92..da4159bc8 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -6,8 +6,8 @@ # example: ynh_mongo_exec --command="db.getMongo().getDBNames().indexOf(\"wekan\")" # # usage: ynh_mongo_exec [--database=database] --command="command" -# | arg: -d, --database= - The database to connect to -# | arg: -c, --command= - The command to evaluate +# | arg: --database= - The database to connect to +# | arg: --command= - The command to evaluate # # ynh_mongo_exec() { @@ -39,7 +39,7 @@ EOF # consider using ynh_mongo_remove_db instead. # # usage: ynh_mongo_drop_db --database=database -# | arg: -d, --database= - The database name to drop +# | arg: --database= - The database name to drop # # ynh_mongo_drop_db() { @@ -57,7 +57,7 @@ ynh_mongo_drop_db() { # example: ynh_mongo_dump_db --database=wekan > ./dump.bson # # usage: ynh_mongo_dump_db --database=database -# | arg: -d, --database= - The database name to dump +# | arg: --database= - The database name to dump # | ret: the mongodump output # # @@ -76,9 +76,9 @@ ynh_mongo_dump_db() { # [internal] # # usage: ynh_mongo_create_user --db_user=user --db_pwd=pwd --db_name=name -# | arg: -u, --db_user= - The user name to create -# | arg: -p, --db_pwd= - The password to identify user by -# | arg: -n, --db_name= - Name of the database to grant privilegies +# | arg: --db_user= - The user name to create +# | arg: --db_pwd= - The password to identify user by +# | arg: --db_name= - Name of the database to grant privilegies # # ynh_mongo_create_user() { @@ -100,7 +100,7 @@ ynh_mongo_create_user() { # Check if a mongo database exists # # usage: ynh_mongo_database_exists --database=database -# | arg: -d, --database= - The database for which to check existence +# | arg: --database= - The database for which to check existence # | exit: Return 1 if the database doesn't exist, 0 otherwise # # @@ -124,7 +124,7 @@ ynh_mongo_database_exists() { # example: ynh_mongo_restore_db --database=wekan < ./dump.bson # # usage: ynh_mongo_restore_db --database=database -# | arg: -d, --database= - The database name to restore +# | arg: --database= - The database name to restore # # ynh_mongo_restore_db() { @@ -142,8 +142,8 @@ ynh_mongo_restore_db() { # [internal] # # usage: ynh_mongo_drop_user --db_user=user --db_name=name -# | arg: -u, --db_user= - The user to drop -# | arg: -n, --db_name= - Name of the database +# | arg: --db_user= - The user to drop +# | arg: --db_name= - Name of the database # # ynh_mongo_drop_user() { @@ -160,9 +160,9 @@ ynh_mongo_drop_user() { # Create a database, an user and its password. Then store the password in the app's config # # usage: ynh_mongo_setup_db --db_user=user --db_name=name [--db_pwd=pwd] -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database -# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# | arg: --db_user= - Owner of the database +# | arg: --db_name= - Name of the database +# | arg: --db_pwd= - Password of the database. If not provided, a password will be generated # # After executing this helper, the password of the created database will be available in $db_pwd # It will also be stored as "mongopwd" into the app settings. @@ -191,8 +191,8 @@ ynh_mongo_setup_db() { # Remove a database if it exists, and the associated user # # usage: ynh_mongo_remove_db --db_user=user --db_name=name -# | arg: -u, --db_user= - Owner of the database -# | arg: -n, --db_name= - Name of the database +# | arg: --db_user= - Owner of the database +# | arg: --db_name= - Name of the database # # ynh_mongo_remove_db() { diff --git a/helpers/helpers.v2.1.d/multimedia b/helpers/helpers.v2.1.d/multimedia index 16f085468..d71346969 100644 --- a/helpers/helpers.v2.1.d/multimedia +++ b/helpers/helpers.v2.1.d/multimedia @@ -6,8 +6,6 @@ readonly MEDIA_DIRECTORY=/home/yunohost.multimedia # Initialize the multimedia directory system # # usage: ynh_multimedia_build_main_dir -# -# Requires YunoHost version 4.2 or higher. ynh_multimedia_build_main_dir() { ## Création du groupe multimedia @@ -55,12 +53,10 @@ ynh_multimedia_build_main_dir() { # # usage: ynh_multimedia_addfolder --source_dir="source_dir" --dest_dir="dest_dir" # -# | arg: -s, --source_dir= - Source directory - The real directory which contains your medias. -# | arg: -d, --dest_dir= - Destination directory - The name and the place of the symbolic link, relative to "/home/yunohost.multimedia" +# | arg: --source_dir= - Source directory - The real directory which contains your medias. +# | arg: --dest_dir= - Destination directory - The name and the place of the symbolic link, relative to "/home/yunohost.multimedia" # # This "directory" will be a symbolic link to a existing directory. -# -# Requires YunoHost version 4.2 or higher. ynh_multimedia_addfolder() { # ============ Argument parsing ============= @@ -86,9 +82,7 @@ ynh_multimedia_addfolder() { # # usage: ynh_multimedia_addaccess user_name # -# | arg: -u, --user_name= - The name of the user which gain this access. -# -# Requires YunoHost version 4.2 or higher. +# | arg: --user_name= - The name of the user which gain this access. ynh_multimedia_addaccess() { # ============ Argument parsing ============= diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql index 15559b2bc..bc532a536 100644 --- a/helpers/helpers.v2.1.d/mysql +++ b/helpers/helpers.v2.1.d/mysql @@ -16,7 +16,7 @@ ynh_mysql_db_shell() { # Create a database and grant optionnaly privilegies to a user # -# [internal] +# [internal] ... handled by the core / "database resource" # # usage: ynh_mysql_create_db db [user [pwd]] # | arg: db - the database name to create @@ -42,7 +42,7 @@ ynh_mysql_create_db() { # Drop a database # -# [internal] +# [internal] ... handled by the core / "database resource" # # If you intend to drop the database *and* the associated user, # consider using ynh_mysql_remove_db instead. @@ -57,7 +57,7 @@ ynh_mysql_drop_db() { # Dump a database # # usage: ynh_mysql_dump_db database -# | arg: -d, --database= - the database name to dump (by default, $db_name) +# | arg: database - the database name to dump (by default, $db_name) # | ret: The mysqldump output # # example: ynh_mysql_dump_db "roundcube" > ./dump.sql @@ -69,7 +69,7 @@ ynh_mysql_dump_db() { # Create a user # -# [internal] +# [internal] ... handled by the core / "database resource" # # usage: ynh_mysql_create_user user pwd [host] # | arg: user - the user name to create @@ -84,7 +84,7 @@ ynh_mysql_create_user() { # [internal] # # usage: ynh_mysql_user_exists user -# | arg: user= - the user for which to check existence +# | arg: user - the user for which to check existence # | ret: 0 if the user exists, 1 otherwise. ynh_mysql_user_exists() { local user=$1 @@ -93,6 +93,8 @@ ynh_mysql_user_exists() { # Check if a mysql database exists # +# [internal] +# # usage: ynh_mysql_database_exists database # | arg: database - the database for which to check existence # | exit: Return 1 if the database doesn't exist, 0 otherwise @@ -104,7 +106,7 @@ ynh_mysql_database_exists() { # Drop a user # -# [internal] +# [internal] ... handled by the core / "database resource" # # usage: ynh_mysql_drop_user user # | arg: user - the user name to drop diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index 59899600c..75fa0f0d4 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -14,8 +14,6 @@ # # This allows to enable/disable specific behaviors dependenging on the install # location -# -# Requires YunoHost version 4.1.0 or higher. ynh_config_add_nginx() { local finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf" @@ -36,8 +34,6 @@ ynh_config_add_nginx() { # Remove the dedicated nginx config # # usage: ynh_config_remove_nginx -# -# Requires YunoHost version 2.7.2 or higher. ynh_config_remove_nginx() { ynh_safe_rm "/etc/nginx/conf.d/$domain.d/$app.conf" ynh_systemctl --service=nginx --action=reload @@ -47,8 +43,6 @@ ynh_config_remove_nginx() { # Regen the nginx config in a change url context # # usage: ynh_config_change_url_nginx -# -# Requires YunoHost version 11.1.9 or higher. ynh_config_change_url_nginx() { # Make a backup of the original NGINX config file if manually modified diff --git a/helpers/helpers.v2.1.d/nodejs b/helpers/helpers.v2.1.d/nodejs index 5e5b13f4e..699288949 100644 --- a/helpers/helpers.v2.1.d/nodejs +++ b/helpers/helpers.v2.1.d/nodejs @@ -4,6 +4,7 @@ readonly N_INSTALL_DIR="/opt/node_n" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. export N_PREFIX="$N_INSTALL_DIR" +# [internal] _ynh_load_nodejs_in_path_and_other_tweaks() { # Get the absolute path of this version of node @@ -27,19 +28,17 @@ _ynh_load_nodejs_in_path_and_other_tweaks() { # Install a specific version of nodejs, using 'n' # -# The installed version is defined by $nodejs_version which should be defined as global prior to calling this helper +# The installed version is defined by `$nodejs_version` which should be defined as global prior to calling this helper # # usage: ynh_nodejs_install # # `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use. # That's how it changes the version # -# Adds the appropriate, specific version of nodejs to the PATH variable (which -# is also exported, to ease the use of ynh_exec_as_app). Also define variable -# PATH_with_nodejs to be used in the systemd config -# (Environment="PATH=__PATH_WITH_NODEJS__") -# -# Requires YunoHost version 2.7.12 or higher. +# The helper adds the appropriate, specific version of nodejs to the `$PATH` variable (which +# is preserved when calling ynh_exec_as_app). Also defines: +# - `$path_with_nodejs` to be used in the systemd config (`Environment="PATH=__PATH_WITH_NODEJS__"`) +# - `$nodejs_dir`, the directory containing the specific version of nodejs, which may be used in the systemd config too (e.g. `ExecStart=__NODEJS_DIR__/node foo bar`) ynh_nodejs_install() { # Use n, https://github.com/tj/n to manage the nodejs versions @@ -105,8 +104,6 @@ ynh_nodejs_install() { # This helper will check if another app uses the same version of node. # - If not, this version of node will be removed. # - If no other app uses node, n will be also removed. -# -# Requires YunoHost version 2.7.12 or higher. ynh_nodejs_remove() { [[ -n "${nodejs_version:-}" ]] || ynh_die "\$nodejs_version should be defined prior to calling ynh_nodejs_remove" diff --git a/helpers/helpers.v2.1.d/permission b/helpers/helpers.v2.1.d/permission index ab74f1f1b..cccba8256 100644 --- a/helpers/helpers.v2.1.d/permission +++ b/helpers/helpers.v2.1.d/permission @@ -27,14 +27,14 @@ # usage: ynh_permission_create --permission="permission" [--url="url"] [--additional_urls="second-url" [ "third-url" ]] [--auth_header=true|false] # [--allowed=group1 [ group2 ]] [--label="label"] [--show_tile=true|false] # [--protected=true|false] -# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) -# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if 'show_tile' is enabled, this URL will be the URL of the tile. -# | arg: -A, --additional_urls= - (optional) List of additional URL for which access will be allowed/forbidden -# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true -# | arg: -a, --allowed= - (optional) A list of group/user to allow for the permission -# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. Default is "APP_LABEL (permission name)". -# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. Defaults to false for the permission different than 'main'. -# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. +# | arg: --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: --url= - (optional) URL for which access will be allowed/forbidden. Note that if 'show_tile' is enabled, this URL will be the URL of the tile. +# | arg: --additional_urls= - (optional) List of additional URL for which access will be allowed/forbidden +# | arg: --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true +# | arg: --allowed= - (optional) A list of group/user to allow for the permission +# | arg: --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. Default is "APP_LABEL (permission name)". +# | arg: --show_tile= - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. Defaults to false for the permission different than 'main'. +# | arg: --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. # # [packagingv1] # @@ -62,9 +62,6 @@ # # Generally this feature is usefull to authenticate automatically the user in the application but in some case the application don't work with theses header and theses header need to be disabled to have the application to work correctly. # See https://github.com/YunoHost/issues/issues/1420 for more informations -# -# -# Requires YunoHost version 3.7.0 or higher. ynh_permission_create() { # ============ Argument parsing ============= local -A args_array=([p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected=) @@ -145,14 +142,10 @@ ynh_permission_create() { # Remove a permission for the app (note that when the app is removed all permission is automatically removed) # -# [packagingv1] -# # example: ynh_permission_delete --permission=editors # # usage: ynh_permission_delete --permission="permission" -# | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) -# -# Requires YunoHost version 3.7.0 or higher. +# | arg: --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) ynh_permission_delete() { # ============ Argument parsing ============= local -A args_array=([p]=permission=) @@ -165,13 +158,9 @@ ynh_permission_delete() { # Check if a permission exists # -# [packagingv1] -# # usage: ynh_permission_exists --permission=permission -# | arg: -p, --permission= - the permission to check +# | arg: --permission= - the permission to check # | exit: Return 1 if the permission doesn't exist, 0 otherwise -# -# Requires YunoHost version 3.7.0 or higher. ynh_permission_exists() { # ============ Argument parsing ============= local -A args_array=([p]=permission=) @@ -185,18 +174,14 @@ ynh_permission_exists() { # Redefine the url associated to a permission # -# [packagingv1] -# # usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]] # [--auth_header=true|false] [--clear_urls] -# | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) -# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if you want to remove url you can pass an empty sting as arguments (""). -# | arg: -a, --add_url= - (optional) List of additional url to add for which access will be allowed/forbidden. -# | arg: -r, --remove_url= - (optional) List of additional url to remove for which access will be allowed/forbidden -# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application -# | arg: -c, --clear_urls - (optional) Clean all urls (url and additional_urls) -# -# Requires YunoHost version 3.7.0 or higher. +# | arg: --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) +# | arg: --url= - (optional) URL for which access will be allowed/forbidden. Note that if you want to remove url you can pass an empty sting as arguments (""). +# | arg: --add_url= - (optional) List of additional url to add for which access will be allowed/forbidden. +# | arg: --remove_url= - (optional) List of additional url to remove for which access will be allowed/forbidden +# | arg: --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application +# | arg: --clear_urls - (optional) Clean all urls (url and additional_urls) ynh_permission_url() { # ============ Argument parsing ============= local -A args_array=([p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls) @@ -255,15 +240,11 @@ ynh_permission_url() { # Update a permission for the app # -# [packagingv1] -# # usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]] # -# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) -# | arg: -a, --add= - the list of group or users to enable add to the permission -# | arg: -r, --remove= - the list of group or users to remove from the permission -# -# Requires YunoHost version 3.7.0 or higher. +# | arg: --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: --add= - the list of group or users to enable add to the permission +# | arg: --remove= - the list of group or users to remove from the permission ynh_permission_update() { # ============ Argument parsing ============= local -A args_array=([p]=permission= [a]=add= [r]=remove=) @@ -302,11 +283,9 @@ ynh_permission_update() { # example: ynh_permission_has_user --permission=main --user=visitors # # usage: ynh_permission_has_user --permission=permission --user=user -# | arg: -p, --permission= - the permission to check -# | arg: -u, --user= - the user seek in the permission +# | arg: --permission= - the permission to check +# | arg: --user= - the user seek in the permission # | exit: Return 1 if the permission doesn't have that user or doesn't exist, 0 otherwise -# -# Requires YunoHost version 3.7.1 or higher. ynh_permission_has_user() { # ============ Argument parsing ============= local -A args_array=([p]=permission= [u]=user=) diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php index a05768015..3cca2c8fb 100644 --- a/helpers/helpers.v2.1.d/php +++ b/helpers/helpers.v2.1.d/php @@ -22,19 +22,19 @@ fi # This will automatically generate an appropriate PHP-FPM configuration for this app. # # The resulting configuration will be deployed to the appropriate place: -# /etc/php/$php_version/fpm/pool.d/$app.conf +# `/etc/php/$php_version/fpm/pool.d/$app.conf` # -# If the app provides a conf/extra_php-fpm.conf template, it will be appended +# If the app provides a `conf/extra_php-fpm.conf` template, it will be appended # to the generated configuration. (In the vast majority of cases, this shouldnt # be necessary) # # $php_version should be defined prior to calling this helper, but there should # be no reason to manually set it, as it is automatically set by the apt # helpers/resources when installing phpX.Y dependencies (PHP apps should at -# least install phpX.Y-fpm using the apt helper/resource) +# least install phpX.Y-fpm using the `apt` helper/resource) # -# $php_group can be defined as a global (from _common.sh) if the worker -# processes should run with a different group than $app +# `$php_group` can be defined as a global (from `_common.sh`) if the worker +# processes should run with a different group than `$app` # # Additional "pm" and "php_admin_value" settings which are meant to be possibly # configurable by admins from a future standard config panel at some point, @@ -46,18 +46,18 @@ fi # defaults than the one set by this helper (while still allowing admin to # override it) you should use `ynh_app_setting_set_default` # -# - $php_upload_max_filezise: corresponds upload_max_filesize and post_max_size. Defaults to 50M -# - $php_process_management: corresponds to "pm" (ondemand, dynamic, static). Defaults to ondemand -# - $php_max_children: by default, computed from "total RAM" divided by 40, cf _default_php_max_children -# - $php_memory_limit: by default, 128M (from global php.ini) +# - `$php_upload_max_filezise`: corresponds upload_max_filesize and post_max_size. Defaults to 50M +# - `$php_process_management`: corresponds to "pm" (ondemand, dynamic, static). Defaults to ondemand +# - `$php_max_children`: by default, computed from "total RAM" divided by 40, cf `_default_php_max_children` +# - `$php_memory_limit`: by default, 128M (from global php.ini) # # Note that if $php_process_management is set to "dynamic", then these # variables MUST be defined prior to calling the helper (no default value) ... # Check PHP-FPM's manual for more info on what these are (: ... # -# - $php_start_servers -# - $php_min_spare_servers -# - $php_max_spare_servers +# - `$php_start_servers` +# - `$php_min_spare_servers` +# - `$php_max_spare_servers` # ynh_config_add_phpfpm() { @@ -135,8 +135,6 @@ EOF # Remove the dedicated PHP-FPM config # # usage: ynh_config_remove_phpfpm -# -# Requires YunoHost version 2.7.2 or higher. ynh_config_remove_phpfpm() { ynh_safe_rm "/etc/php/$php_version/fpm/pool.d/$app.conf" ynh_systemctl --service="php${php_version}-fpm" --action=reload @@ -161,51 +159,3 @@ _default_php_max_children() { echo "$php_max_children" } - - -# Execute a command with Composer -# -# Will use $install_dir as workdir unless $composer_workdir exists (but that shouldnt be necessary) -# -# You may also define composer_user=root prior to call this helper if you absolutely need composer to run as root, but this is discouraged... -# -# usage: ynh_composer_exec commands -# -# Requires YunoHost version 4.2 or higher. -ynh_composer_exec() { - local workdir="${composer_workdir:-$install_dir}" - - COMPOSER_HOME="$workdir/.composer" \ - COMPOSER_MEMORY_LIMIT=-1 \ - sudo -E -u "${composer_user:-$app}" \ - php${php_version} "$workdir/composer.phar" $@ \ - -d "$workdir" --no-interaction --no-ansi 2>&1 -} - -# Install and initialize Composer in the given directory -# -# The installed version is defined by $composer_version which should be defined -# as global prior to calling this helper. -# -# Will use $install_dir as workdir unless $composer_workdir exists (but that shouldnt be necessary) -# -# usage: ynh_composer_install -# -# Requires YunoHost version 4.2 or higher. -ynh_composer_install() { - local workdir="${composer_workdir:-$install_dir}" - - [[ -n "${composer_version}" ]] || ynh_die "\$composer_version should be defined before calling ynh_composer_install. (In the past, this was called \$YNH_COMPOSER_VERSION)" - - [[ ! -e "$workdir/composer.phar" ]] || ynh_safe_rm $workdir/composer.phar - - local composer_url="https://getcomposer.org/download/$composer_version/composer.phar" - - # NB. we have to declare the var as local first, - # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work - # because local always return 0 ... - local out - # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) - out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir/composer.phar $composer_url 2>&1) \ - || ynh_die "$out" -} diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql index ffaeafbf3..2b0b55bf4 100644 --- a/helpers/helpers.v2.1.d/postgresql +++ b/helpers/helpers.v2.1.d/postgresql @@ -19,7 +19,7 @@ ynh_psql_db_shell() { # Create a database and grant optionnaly privilegies to a user # -# [internal] +# [internal] ... handled by the core / "database resource" # # usage: ynh_psql_create_db db [user] # | arg: db - the database name to create @@ -42,7 +42,7 @@ ynh_psql_create_db() { # Drop a database # -# [internal] +# [internal] ... handled by the core / "database resource" # # If you intend to drop the database *and* the associated user, # consider using ynh_psql_remove_db instead. @@ -74,7 +74,7 @@ ynh_psql_dump_db() { # Create a user # -# [internal] +# [internal] ... handled by the core / "database resource" # # usage: ynh_psql_create_user user pwd # | arg: user - the user name to create @@ -88,7 +88,7 @@ ynh_psql_create_user() { # Check if a psql user exists # -# [packagingv1] +# [internal] # # usage: ynh_psql_user_exists user # | arg: user= - the user for which to check existence @@ -101,6 +101,8 @@ ynh_psql_user_exists() { # Check if a psql database exists # +# [internal] +# # usage: ynh_psql_database_exists database # | arg: database - the database for which to check existence # | exit: Return 1 if the database doesn't exist, 0 otherwise @@ -112,7 +114,7 @@ ynh_psql_database_exists() { # Drop a user # -# [internal] +# [internal] ... handled by the core / "database resource" # # usage: ynh_psql_drop_user user # | arg: user - the user name to drop diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby index c33967a9f..22b28d9ad 100644 --- a/helpers/helpers.v2.1.d/ruby +++ b/helpers/helpers.v2.1.d/ruby @@ -31,24 +31,16 @@ _ynh_load_ruby_in_path_and_other_tweaks() { # Install a specific version of Ruby using rbenv # -# The installed version is defined by $ruby_version which should be defined as global prior to calling this helper -# -# This helper creates a /etc/profile.d/rbenv.sh that configures PATH environment for rbenv -# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) -# -# Don't forget to execute ruby-dependent command in a login environment -# (e.g. sudo --login option) -# When not possible (e.g. in systemd service definition), please use direct path -# to rbenv shims (e.g. $RBENV_ROOT/shims/bundle) +# The installed version is defined by `$ruby_version` which should be defined as global prior to calling this helper # # usage: ynh_ruby_install # -# Adds the appropriate, specific version of ruby to the PATH variable (which -# is also exported, to ease the use of ynh_exec_as_app). Also define variable -# PATH_with_ruby to be used in the systemd config -# (Environment="PATH=__PATH_WITH_RUBY__") +# The helper adds the appropriate, specific version of ruby to the `$PATH` variable (which +# is preserved when calling ynh_exec_as_app). Also defines: +# - `$path_with_ruby` to be used in the systemd config (`Environment="PATH=__PATH_WITH_RUBY__"`) +# - `$ruby_dir`, the directory containing the specific version of ruby, which may be used in the systemd config too (e.g. `ExecStart=__RUBY_DIR__/ruby foo bar`) # -# Requires YunoHost version 3.2.2 or higher. +# This helper also creates a /etc/profile.d/rbenv.sh that configures PATH environment for rbenv ynh_ruby_install () { [[ -n "${ruby_version:-}" ]] || ynh_die "\$ruby_version should be defined prior to calling ynh_ruby_install" @@ -70,7 +62,7 @@ ynh_ruby_install () { if git remote -v 2>/dev/null | grep "https://github.com/rbenv/rbenv.git"; then echo "Updating rbenv..." git pull -q --tags origin master - ynh_ruby_try_bash_extension + _ynh_ruby_try_bash_extension else echo "Reinstalling rbenv..." cd .. @@ -80,7 +72,7 @@ ynh_ruby_install () { git init -q git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 git checkout -q -b master origin/master - ynh_ruby_try_bash_extension + _ynh_ruby_try_bash_extension rbenv=$RBENV_INSTALL_DIR/bin/rbenv fi popd @@ -90,7 +82,7 @@ ynh_ruby_install () { git init -q git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 git checkout -q -b master origin/master - ynh_ruby_try_bash_extension + _ynh_ruby_try_bash_extension rbenv=$RBENV_INSTALL_DIR/bin/rbenv popd fi @@ -187,7 +179,7 @@ eval \"\$(rbenv init -)\" # Remove the version of Ruby used by the app. # -# This helper will also cleanup Ruby versions +# This helper will also cleanup unused Ruby versions # # usage: ynh_ruby_remove ynh_ruby_remove () { @@ -211,6 +203,8 @@ ynh_ruby_remove () { # Remove no more needed versions of Ruby used by the app. # +# [internal] +# # This helper will check what Ruby version are no more required, # and uninstall them # If no app uses Ruby, rbenv will be also removed. @@ -249,7 +243,7 @@ _ynh_ruby_cleanup () { fi } -ynh_ruby_try_bash_extension() { +_ynh_ruby_try_bash_extension() { if [ -x src/configure ]; then src/configure && make -C src 2>&1 || { ynh_print_info "Optional bash extension failed to build, but things will still work normally." diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting index d8053066d..82528efa5 100644 --- a/helpers/helpers.v2.1.d/setting +++ b/helpers/helpers.v2.1.d/setting @@ -2,11 +2,9 @@ # Get an application setting # -# usage: ynh_app_setting_get --app=app --key=key -# | arg: -a, --app= - the application id -# | arg: -k, --key= - the setting to get -# -# Requires YunoHost version 2.2.4 or higher. +# usage: ynh_app_setting_get --key=key +# | arg: --app= - the application id (global $app by default) +# | arg: --key= - the setting to get ynh_app_setting_get() { # ============ Argument parsing ============= local _globalapp=${app-:} @@ -22,12 +20,10 @@ ynh_app_setting_get() { # Set an application setting # -# usage: ynh_app_setting_set --app=app --key=key --value=value -# | arg: -a, --app= - the application id -# | arg: -k, --key= - the setting name to set -# | arg: -v, --value= - the setting value to set -# -# Requires YunoHost version 2.2.4 or higher. +# usage: ynh_app_setting_set --key=key --value=value +# | arg: --app= - the application id (global $app by default) +# | arg: --key= - the setting name to set +# | arg: --value= - the setting value to set ynh_app_setting_set() { # ============ Argument parsing ============= local _globalapp=${app-:} @@ -54,12 +50,10 @@ ynh_app_setting_set() { # ynh_app_setting_set --key="foo" --value="$foo" # fi # -# usage: ynh_app_setting_set_default --app=app --key=key --value=value -# | arg: -a, --app= - the application id -# | arg: -k, --key= - the setting name to set -# | arg: -v, --value= - the default setting value to set -# -# Requires YunoHost version 11.1.16 or higher. +# usage: ynh_app_setting_set_default --key=key --value=value +# | arg: --app= - the application id (global $app by default) +# | arg: --key= - the setting name to set +# | arg: --value= - the default setting value to set ynh_app_setting_set_default() { # ============ Argument parsing ============= local _globalapp=${app-:} @@ -79,11 +73,9 @@ ynh_app_setting_set_default() { # Delete an application setting # -# usage: ynh_app_setting_delete --app=app --key=key -# | arg: -a, --app= - the application id -# | arg: -k, --key= - the setting to delete -# -# Requires YunoHost version 2.2.4 or higher. +# usage: ynh_app_setting_delete --key=key +# | arg: --app= - the application id (global $app by default) +# | arg: --key= - the setting to delete ynh_app_setting_delete() { # ============ Argument parsing ============= local _globalapp=${app-:} diff --git a/helpers/helpers.v2.1.d/sources b/helpers/helpers.v2.1.d/sources new file mode 100644 index 000000000..ced0e05c7 --- /dev/null +++ b/helpers/helpers.v2.1.d/sources @@ -0,0 +1,261 @@ +#!/bin/bash + +# Download, check integrity, uncompress and patch upstream sources +# +# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] +# | arg: --dest_dir= - Directory where to setup sources +# | arg: --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise +# | arg: --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders) +# | arg: --full_replace= - Remove previous sources before installing new sources (can be 1 or 0, default to 0) +# +# ##### New 'sources' resources +# +# (See also the resources documentation which may be more complete?) +# +# This helper will read infos from the 'sources' resources in the manifest.toml of the app +# and expect a structure like: +# +# ```toml +# [resources.sources] +# [resources.sources.main] +# url = "https://some.address.to/download/the/app/archive" +# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL +# ``` +# +# ##### Optional flags +# +# ```text +# format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract +# "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract +# "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract +# "whatever" # an arbitrary value, not really meaningful except to imply that the file won't be extracted +# +# in_subdir = true # default, there's an intermediate subdir in the archive before accessing the actual files +# false # sources are directly in the archive root +# n # (special cases) an integer representing a number of subdirs levels to get rid of +# +# extract = true # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ... +# = false # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset. +# # in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value +# +# rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical +# platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for +# ``` +# +# You may also define assets url and checksum per-architectures such as: +# ```toml +# [resources.sources] +# [resources.sources.main] +# amd64.url = "https://some.address.to/download/the/app/archive/when/amd64" +# amd64.sha256 = "0123456789abcdef" +# armhf.url = "https://some.address.to/download/the/app/archive/when/armhf" +# armhf.sha256 = "fedcba9876543210" +# ``` +# +# In which case `ynh_setup_source --dest_dir="$install_dir"` will automatically pick the appropriate source depending on the arch +# +# The helper will: +# - Download the specific URL if there is no local archive +# - Check the integrity with the specific sha256 sum +# - Uncompress the archive to `$dest_dir`. +# - If `in_subdir` is true, the first level directory of the archive will be removed. +# - If `in_subdir` is a numeric value, the N first level directories will be removed. +# - Patches named `patches/${src_id}-*.patch` will be applied to `$dest_dir` +ynh_setup_source() { + # ============ Argument parsing ============= + local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace) + local dest_dir + local source_id + local keep + local full_replace + ynh_handle_getopts_args "$@" + keep="${keep:-}" + full_replace="${full_replace:-0}" + source_id="${source_id:-main}" + # =========================================== + + local sources_json=$(ynh_read_manifest "resources.sources[\"$source_id\"]") + if jq -re ".url" <<< "$sources_json" + then + local arch_prefix="" + else + local arch_prefix=".$YNH_ARCH" + fi + + local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" + local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" + local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" + local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" + src_in_subdir=${src_in_subdir:-true} + local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" + local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" + local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" + + [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" + + if [[ -z "$src_format" ]] + then + if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] + then + src_format="zip" + elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] + then + src_format="tar.gz" + elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] + then + src_format="tar.xz" + elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] + then + src_format="tar.bz2" + elif [[ -z "$src_extract" ]] + then + src_extract="false" + fi + fi + + src_format=${src_format:-tar.gz} + src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') + src_extract=${src_extract:-true} + + if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] + then + ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" + fi + + # Gotta use this trick with 'dirname' because source_id may contain slashes x_x + mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) + src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" + + if [ "$src_format" = "docker" ]; then + src_platform="${src_platform:-"linux/$YNH_ARCH"}" + else + [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" + + # If the file was prefetched but somehow doesn't match the sum, rm and redownload it + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status + then + rm -f "$src_filename" + fi + + # Only redownload the file if it wasnt prefetched + if [ ! -e "$src_filename" ] + then + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ + || ynh_die "$out" + fi + + # Check the control sum + if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status + then + local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_size="$(du -hs ${src_filename} | cut --fields=1)" + rm -f ${src_filename} + ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." + fi + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ + if [ -n "$keep" ] && [ -e "$dest_dir" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + mkdir -p $keep_dir + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$dest_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" + cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" + fi + done + fi + + if [ "$full_replace" -eq 1 ]; then + ynh_safe_rm "$dest_dir" + fi + + # Extract source into the app dir + mkdir --parents "$dest_dir" + + if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then + _ynh_apply_default_permissions $dest_dir + fi + + if [[ "$src_extract" == "false" ]]; then + if [[ -z "$src_rename" ]] + then + mv $src_filename $dest_dir + else + mv $src_filename $dest_dir/$src_rename + fi + elif [[ "$src_format" == "docker" ]]; then + "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1 + elif [[ "$src_format" == "zip" ]]; then + # Zip format + # Using of a temp directory, because unzip doesn't manage --strip-components + if $src_in_subdir; then + local tmp_dir=$(mktemp --directory) + unzip -quo $src_filename -d "$tmp_dir" + cp --archive $tmp_dir/*/. "$dest_dir" + ynh_safe_rm "$tmp_dir" + else + unzip -quo $src_filename -d "$dest_dir" + fi + ynh_safe_rm "$src_filename" + else + local strip="" + if [ "$src_in_subdir" != "false" ]; then + if [ "$src_in_subdir" == "true" ]; then + local sub_dirs=1 + else + local sub_dirs="$src_in_subdir" + fi + strip="--strip-components $sub_dirs" + fi + if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then + tar --extract --file=$src_filename --directory="$dest_dir" $strip + else + ynh_die "Archive format unrecognized." + fi + ynh_safe_rm "$src_filename" + fi + + # Apply patches + if [ -d "$YNH_APP_BASEDIR/patches/" ]; then + local patches_folder=$(realpath $YNH_APP_BASEDIR/patches/) + # Check if any file matching the pattern exists, cf https://stackoverflow.com/a/34195247 + if compgen -G "$patches_folder/${source_id}-*.patch" >/dev/null; then + pushd "$dest_dir" + for p in $patches_folder/${source_id}-*.patch; do + echo $p + patch --strip=1 <$p || ynh_print_warn "Packagers /!\\ patch $p failed to apply" + done + popd + fi + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + if [ -n "$keep" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$keep_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" + + # We add "--no-target-directory" (short option is -T) to handle the special case + # when we "keep" a folder, but then the new setup already contains the same dir (but possibly empty) + # in which case a regular "cp" will create a copy of the directory inside the directory ... + # resulting in something like /var/www/$app/data/data instead of /var/www/$app/data + # cf https://unix.stackexchange.com/q/94831 for a more elaborate explanation on the option + cp --archive --no-target-directory "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" + fi + done + fi + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ +} diff --git a/helpers/helpers.v2.1.d/string b/helpers/helpers.v2.1.d/string index 05c8a8cd0..d03fcff19 100644 --- a/helpers/helpers.v2.1.d/string +++ b/helpers/helpers.v2.1.d/string @@ -3,13 +3,11 @@ # Generate a random string # # usage: ynh_string_random [--length=string_length] -# | arg: -l, --length= - the string length to generate (default: 24) -# | arg: -f, --filter= - the kind of characters accepted in the output (default: 'A-Za-z0-9') +# | arg: --length= - the string length to generate (default: 24) +# | arg: --filter= - the kind of characters accepted in the output (default: 'A-Za-z0-9') # | ret: the generated string # # example: pwd=$(ynh_string_random --length=8) -# -# Requires YunoHost version 2.2.4 or higher. ynh_string_random() { # ============ Argument parsing ============= local -A args_array=([l]=length= [f]=filter=) @@ -28,14 +26,12 @@ ynh_string_random() { # Substitute/replace a string (or expression) by another in a file # # usage: ynh_replace --match=match --replace=replace --file=file -# | arg: -m, --match= - String to be searched and replaced in the file -# | arg: -r, --replace= - String that will replace matches -# | arg: -f, --file= - File in which the string will be replaced. +# | arg: --match= - String to be searched and replaced in the file +# | arg: --replace= - String that will replace matches +# | arg: --file= - File in which the string will be replaced. # # As this helper is based on sed command, regular expressions and references to # sub-expressions can be used (see sed manual page for more information) -# -# Requires YunoHost version 2.6.4 or higher. ynh_replace() { # ============ Argument parsing ============= local -A args_array=([m]=match= [r]=replace= [f]=file=) @@ -55,18 +51,16 @@ ynh_replace() { sed --in-place "s${delimit}${match}${delimit}${replace}${delimit}g" "$file" } -# Substitute/replace a special string by another in a file +# Substitute/replace a regex in a file # -# usage: ynh_replace_special_string --match=match --replace=replace --file=file -# | arg: -m, --match= - String to be searched and replaced in the file -# | arg: -r, --replace= - String that will replace matches -# | arg: -f, --file= - File in which the string will be replaced. +# usage: ynh_replace_regex --match=match --replace=replace --file=file +# | arg: --match= - String to be searched and replaced in the file +# | arg: --replace= - String that will replace matches +# | arg: --file= - File in which the string will be replaced. # # This helper will use ynh_replace, but as you can use special # characters, you can't use some regular expressions and sub-expressions. -# -# Requires YunoHost version 2.7.7 or higher. -ynh_replace_special_string() { +ynh_replace_regex() { # ============ Argument parsing ============= local -A args_array=([m]=match= [r]=replace= [f]=file=) local match @@ -91,14 +85,12 @@ ynh_replace_special_string() { # [packagingv1] # # usage: ynh_sanitize_dbid --db_name=name -# | arg: -n, --db_name= - name to correct/sanitize +# | arg: --db_name= - name to correct/sanitize # | ret: the corrected name # # example: dbname=$(ynh_sanitize_dbid $app) # # Underscorify the string (replace - and . by _) -# -# Requires YunoHost version 2.2.4 or higher. ynh_sanitize_dbid() { # ============ Argument parsing ============= local -A args_array=([n]=db_name=) @@ -123,8 +115,6 @@ ynh_sanitize_dbid() { # ynh_normalize_url_path / # -> / # # usage: ynh_normalize_url_path path_to_normalize -# -# Requires YunoHost version 2.6.4 or higher. ynh_normalize_url_path() { local path_url=$1 diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd index 0c0be3e67..b8ff84fd1 100644 --- a/helpers/helpers.v2.1.d/systemd +++ b/helpers/helpers.v2.1.d/systemd @@ -3,15 +3,13 @@ # Create a dedicated systemd config # # usage: ynh_config_add_systemd [--service=service] [--template=template] -# | arg: -s, --service= - Service name (optionnal, `$app` by default) -# | arg: -t, --template= - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template) +# | arg: --service= - Service name (optionnal, `$app` by default) +# | arg: --template= - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template) # # This will use the template `../conf/.service`. # # See the documentation of `ynh_config_add` for a description of the template # format and how placeholders are replaced with actual variables. -# -# Requires YunoHost version 4.1.0 or higher. ynh_config_add_systemd() { # ============ Argument parsing ============= local -A args_array=([s]=service= [t]=template=) @@ -45,14 +43,12 @@ ynh_config_remove_systemd() { # Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started # # usage: ynh_systemctl [--service=service] [--action=action] [ [--wait_until="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] -# | arg: -n, --service= - Name of the service to start. Default : `$app` -# | arg: -a, --action= - Action to perform with systemctl. Default: start -# | arg: -w, --wait_until= - The pattern to find in the log to attest the service is effectively fully started. -# | arg: -p, --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` -# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 60 seconds. -# | arg: -e, --length= - Length of the error log displayed for debugging : Default : 20 -# -# Requires YunoHost version 3.5.0 or higher. +# | arg: --service= - Name of the service to start. Default : `$app` +# | arg: --action= - Action to perform with systemctl. Default: start +# | arg: --wait_until= - The pattern to find in the log to attest the service is effectively fully started. +# | arg: --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` +# | arg: --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 60 seconds. +# | arg: --length= - Length of the error log displayed for debugging : Default : 20 ynh_systemctl() { # ============ Argument parsing ============= local -A args_array=([n]=service= [a]=action= [w]=wait_until= [p]=log_path= [t]=timeout= [e]=length=) diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/systemuser similarity index 52% rename from helpers/helpers.v2.1.d/user rename to helpers/helpers.v2.1.d/systemuser index 1ce0e78de..720a6ec28 100644 --- a/helpers/helpers.v2.1.d/user +++ b/helpers/helpers.v2.1.d/systemuser @@ -1,66 +1,10 @@ #!/bin/bash -# Check if a YunoHost user exists -# -# usage: ynh_user_exists --username=username -# | arg: -u, --username= - the username to check -# | ret: 0 if the user exists, 1 otherwise. -# -# example: ynh_user_exists 'toto' || echo "User does not exist" -# -# Requires YunoHost version 2.2.4 or higher. -ynh_user_exists() { - # ============ Argument parsing ============= - local -A args_array=([u]=username=) - local username - ynh_handle_getopts_args "$@" - # =========================================== - - yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null -} - -# Retrieve a YunoHost user information -# -# usage: ynh_user_get_info --username=username --key=key -# | arg: -u, --username= - the username to retrieve info from -# | arg: -k, --key= - the key to retrieve -# | ret: the value associate to that key -# -# example: mail=$(ynh_user_get_info --username="toto" --key=mail) -# -# Requires YunoHost version 2.2.4 or higher. -ynh_user_get_info() { - # ============ Argument parsing ============= - local -A args_array=([u]=username= [k]=key=) - local username - local key - ynh_handle_getopts_args "$@" - # =========================================== - - yunohost user info "$username" --output-as json --quiet | jq -r ".$key" -} - -# Get the list of YunoHost users -# -# usage: ynh_user_list -# | ret: one username per line as strings -# -# example: for u in $(ynh_user_list); do ... ; done -# -# Requires YunoHost version 2.4.0 or higher. -ynh_user_list() { - yunohost user list --output-as json --quiet | jq -r ".users | keys[]" -} - # Check if a user exists on the system # -# [packagingv1] -# # usage: ynh_system_user_exists --username=username -# | arg: -u, --username= - the username to check +# | arg: --username= - the username to check # | ret: 0 if the user exists, 1 otherwise. -# -# Requires YunoHost version 2.2.4 or higher. ynh_system_user_exists() { # ============ Argument parsing ============= local -A args_array=([u]=username=) @@ -73,13 +17,9 @@ ynh_system_user_exists() { # Check if a group exists on the system # -# [packagingv1] -# # usage: ynh_system_group_exists --group=group -# | arg: -g, --group= - the group to check +# | arg: --group= - the group to check # | ret: 0 if the group exists, 1 otherwise. -# -# Requires YunoHost version 3.5.0.2 or higher. ynh_system_group_exists() { # ============ Argument parsing ============= local -A args_array=([g]=group=) @@ -92,13 +32,11 @@ ynh_system_group_exists() { # Create a system user # -# [packagingv1] -# # usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell] [--groups="group1 group2"] -# | arg: -u, --username= - Name of the system user that will be create -# | arg: -h, --home_dir= - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home -# | arg: -s, --use_shell - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell -# | arg: -g, --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp) +# | arg: --username= - Name of the system user that will be create +# | arg: --home_dir= - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home +# | arg: --use_shell - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell +# | arg: --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp) # # Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability) : # ``` @@ -108,8 +46,6 @@ ynh_system_group_exists() { # ``` # ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell # ``` -# -# Requires YunoHost version 2.6.4 or higher. ynh_system_user_create() { # ============ Argument parsing ============= local -A args_array=([u]=username= [h]=home_dir= [s]=use_shell [g]=groups=) @@ -146,12 +82,8 @@ ynh_system_user_create() { # Delete a system user # -# [packagingv1] -# # usage: ynh_system_user_delete --username=user_name -# | arg: -u, --username= - Name of the system user that will be create -# -# Requires YunoHost version 2.6.4 or higher. +# | arg: --username= - Name of the system user that will be create ynh_system_user_delete() { # ============ Argument parsing ============= local -A args_array=([u]=username=) @@ -171,12 +103,3 @@ ynh_system_user_delete() { delgroup $username fi } - -# Execute a command after sudoing as $app -# -# Note that the $PATH variable is preserved (using --preserve-env=PATH) -# -# usage: ynh_exec_as_app COMMAND [ARG ...] -ynh_exec_as_app() { - sudo --preserve-env=PATH -u "$app" "$@" -} diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index b7dc3bc4a..ffd161278 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -3,75 +3,44 @@ # Create a dedicated config file from a template # # usage: ynh_config_add --template="template" --destination="destination" -# | arg: -t, --template= - Template config file to use -# | arg: -d, --destination= - Destination of the config file -# | arg: -j, --jinja - Use jinja template instead of legacy __MY_VAR__ +# | arg: --template= - Template config file to use +# | arg: --destination= - Destination of the config file +# | arg: --jinja - Use jinja template instead of the simple `__MY_VAR__` templating format # # examples: -# ynh_config_add --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" -# ynh_config_add --jinja --template="config.j2" --destination="$install_dir/config" use the template file "../conf/config.j2" -# ynh_config_add --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" +# ynh_add_config --template=".env" --destination="$install_dir/.env" # (use the template file "conf/.env" from the app's package) +# ynh_add_config --jinja --template="config.j2" --destination="$install_dir/config" # (use the template file "conf/config.j2" from the app's package) # -## -## How it works in "legacy" mode -## -# The template can be by default the name of a file in the conf directory -# of a YunoHost Package, a relative path or an absolute path. +# The template can be 1) the name of a file in the `conf` directory of +# the app, 2) a relative path or 3) an absolute path. # -# The helper will use the template `template` to generate a config file -# `destination` by replacing the following keywords with global variables -# that should be defined before calling this helper : -# ``` -# __USER__ by $app -# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH -# ``` -# And any dynamic variables that should be defined before calling this helper like: -# ``` -# __DOMAIN__ by $domain -# __PATH__ by $path -# __APP__ by $app -# __VAR_1__ by $var_1 -# __VAR_2__ by $var_2 -# ``` +# This applies a simple templating format which covers a good 95% of cases, +# where patterns like `__FOO__` are replaced by the bash variable `$foo`, for example: +# `__DOMAIN__` by `$domain` +# `__PATH__` by `$path` +# `__APP__` by `$app` +# `__VAR_1__` by `$var_1` +# `__VAR_2__` by `$var_2` # -## -## When --jinja is enabled -## -# For a full documentation of the template you can refer to: https://jinja.palletsprojects.com/en/3.1.x/templates/ -# In Yunohost context there are no really some specificity except that all variable passed are of type string. -# So here are some example of recommended usage: +# Special case for `__PATH__/` which is replaced by `/` instead of `//` if `$path` is `/` # -# If you need a conditional block +# ##### When --jinja is enabled # -# {% if should_my_block_be_shown == 'true' %} -# ... -# {% endif %} +# This option is meant for advanced use-cases where the "simple" templating +# mode ain't enough because you need conditional blocks or loops. # -# or +# For a full documentation of jinja's syntax you can refer to: +# https://jinja.palletsprojects.com/en/3.1.x/templates/ # -# {% if should_my_block_be_shown == '1' %} -# ... -# {% endif %} +# Note that in YunoHost context, all variables are from shell variables and therefore are strings # -# If you need to iterate with loop: -# -# {% for yolo in var_with_multiline_value.splitlines() %} -# ... -# {% endfor %} -# -# or -# -# {% for jail in my_var_with_coma.split(',') %} -# ... -# {% endfor %} +# ##### Keeping track of manual changes by the admin # # The helper will verify the checksum and backup the destination file # if it's different before applying the new template. # # And it will calculate and store the destination file checksum # into the app settings when configuration is done. -# -# Requires YunoHost version 4.1.0 or higher. ynh_config_add() { # ============ Argument parsing ============= local -A args_array=([t]=template= [d]=destination= [j]=jinja) @@ -110,61 +79,41 @@ ynh_config_add() { ) else cp -f "$template_path" "$destination" - ynh_replace_vars --file="$destination" + _ynh_replace_vars "$destination" fi ynh_store_file_checksum "$destination" } -# Replace variables in a file +# Replace `__FOO__` patterns in file with bash variable `$foo` # # [internal] # -# usage: ynh_replace_vars --file="file" -# | arg: -f, --file= - File where to replace variables +# usage: ynh_replace_vars "/path/to/file" +# | arg: /path/to/file - File where to replace variables # -# The helper will replace the following keywords with global variables -# that should be defined before calling this helper : -# __PATH__ by $path -# __PATH__/ by $path/ if $path != /, or just / otherwise (instead of //) -# __USER__ by $app -# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# This applies a simple templating format which covers a good 95% of cases, +# where patterns like `__FOO__` are replaced by the bash variable `$foo`, for example: +# `__DOMAIN__` by `$domain` +# `__PATH__` by `$path` +# `__APP__` by `$app` +# `__VAR_1__` by `$var_1` +# `__VAR_2__` by `$var_2` # -# And any dynamic variables that should be defined before calling this helper like: -# __DOMAIN__ by $domain -# __APP__ by $app -# __VAR_1__ by $var_1 -# __VAR_2__ by $var_2 -# -# Requires YunoHost version 4.1.0 or higher. -ynh_replace_vars() { - # ============ Argument parsing ============= - local -A args_array=([f]=file=) - local file - ynh_handle_getopts_args "$@" - # =========================================== +# Special case for `__PATH__/` which is replaced by `/` instead of `//` if `$path` is `/` +_ynh_replace_vars() { + local file=$1 - # Replace specific YunoHost variables - if test -n "${path:-}"; then - # path_slash_less is path, or a blank value if path is only '/' - local path_slash_less=${path%/} - ynh_replace --match="__PATH__/" --replace="$path_slash_less/" --file="$file" - ynh_replace --match="__PATH__" --replace="$path" --file="$file" - fi - if test -n "${app:-}"; then - ynh_replace --match="__USER__" --replace="$app" --file="$file" - fi - if test -n "${ynh_node_load_PATH:-}"; then - ynh_replace --match="__YNH_NODE_LOAD_PATH__" --replace="$ynh_node_load_PATH" --file="$file" - fi - - # Replace others variables - - # List other unique (__ __) variables in $file + # List unique (__ __) variables in $file local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g")) set +o xtrace # set +x + # Specific trick to make sure that __PATH__/ doesn't end up in "//" if $path=/ + if [[ "${path:-}" == "/" ]] && grep -q '__PATH__/' $file; then + sed --in-place "s@__PATH__/@${path%/}@g" "$file" + fi + # Do the replacement local delimit=@ for one_var in "${uniques_vars[@]}"; do @@ -191,9 +140,9 @@ ynh_replace_vars() { # Get a value from heterogeneous file (yaml, json, php, python...) # # usage: ynh_read_var_in_file --file=PATH --key=KEY -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to get -# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# | arg: --file= - the path to the file +# | arg: --key= - the key to get +# | arg: --after= - the line just before the key (in case of multiple lines with the name of the key in the file) # # This helpers match several var affectation use case in several languages # We don't use jq or equivalent to keep comments and blank space in files @@ -220,8 +169,6 @@ ynh_replace_vars() { # USER = 8102 # user = 'https://donate.local' # CUSTOM['user'] = 'YunoHost' -# -# Requires YunoHost version 4.3 or higher. ynh_read_var_in_file() { # ============ Argument parsing ============= local -A args_array=([f]=file= [k]=key= [a]=after=) @@ -293,12 +240,10 @@ ynh_read_var_in_file() { # Set a value into heterogeneous file (yaml, json, php, python...) # # usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE -# | arg: -f, --file= - the path to the file -# | arg: -k, --key= - the key to set -# | arg: -v, --value= - the value to set -# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) -# -# Requires YunoHost version 4.3 or higher. +# | arg: --file= - the path to the file +# | arg: --key= - the key to set +# | arg: --value= - the value to set +# | arg: --after= - the line just before the key (in case of multiple lines with the name of the key in the file) ynh_write_var_in_file() { # ============ Argument parsing ============= local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index c46c41d7c..be57a6d73 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -17,8 +17,6 @@ YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)} # This function provide a way to clean some residual of installation that not managed by remove script. # # It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script -# -# Requires YunoHost version 2.6.4 or higher. ynh_exit_properly() { local exit_code=$? @@ -58,8 +56,6 @@ ynh_exit_properly() { # This configure the rest of the script execution such that, if an error occurs # or if an empty variable is used, the execution of the script stops immediately # and a call to `ynh_clean_setup` is triggered if it has been defined by your script. -# -# Requires YunoHost version 2.6.4 or higher. ynh_abort_if_errors() { set -o errexit # set -e; Exit if a command fail set -o nounset # set -u; And if a variable is used unset @@ -72,266 +68,13 @@ then ynh_abort_if_errors fi -# Download, check integrity, uncompress and patch upstream sources +# Execute a command after sudoing as $app # -# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] -# | arg: -d, --dest_dir= - Directory where to setup sources -# | arg: -s, --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise -# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders) -# | arg: -r, --full_replace= - Remove previous sources before installing new sources (can be 1 or 0, default to 0) +# Note that the $PATH variable is preserved (using --preserve-env=PATH) # -# #### New 'sources' resources -# -# (See also the resources documentation which may be more complete?) -# -# This helper will read infos from the 'sources' resources in the manifest.toml of the app -# and expect a structure like: -# -# ```toml -# [resources.sources] -# [resources.sources.main] -# url = "https://some.address.to/download/the/app/archive" -# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL -# ``` -# -# ##### Optional flags -# -# ```text -# format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract -# "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract -# "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract -# "whatever" # an arbitrary value, not really meaningful except to imply that the file won't be extracted -# -# in_subdir = true # default, there's an intermediate subdir in the archive before accessing the actual files -# false # sources are directly in the archive root -# n # (special cases) an integer representing a number of subdirs levels to get rid of -# -# extract = true # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ... -# = false # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset. -# # in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value -# -# rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical -# platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for -# ``` -# -# You may also define assets url and checksum per-architectures such as: -# ```toml -# [resources.sources] -# [resources.sources.main] -# amd64.url = "https://some.address.to/download/the/app/archive/when/amd64" -# amd64.sha256 = "0123456789abcdef" -# armhf.url = "https://some.address.to/download/the/app/archive/when/armhf" -# armhf.sha256 = "fedcba9876543210" -# ``` -# -# In which case ynh_setup_source --dest_dir="$install_dir" will automatically pick the appropriate source depending on the arch -# -# The helper will: -# - Download the specific URL if there is no local archive -# - Check the integrity with the specific sha256 sum -# - Uncompress the archive to `$dest_dir`. -# - If `in_subdir` is true, the first level directory of the archive will be removed. -# - If `in_subdir` is a numeric value, the N first level directories will be removed. -# - Patches named `patches/${src_id}-*.patch` will be applied to `$dest_dir` -# -# Requires YunoHost version 2.6.4 or higher. -ynh_setup_source() { - # ============ Argument parsing ============= - local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace) - local dest_dir - local source_id - local keep - local full_replace - ynh_handle_getopts_args "$@" - keep="${keep:-}" - full_replace="${full_replace:-0}" - source_id="${source_id:-main}" - # =========================================== - - local sources_json=$(ynh_read_manifest "resources.sources[\"$source_id\"]") - if jq -re ".url" <<< "$sources_json" - then - local arch_prefix="" - else - local arch_prefix=".$YNH_ARCH" - fi - - local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" - local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" - local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" - local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" - src_in_subdir=${src_in_subdir:-true} - local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" - local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" - local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" - - [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" - [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" - - if [[ -z "$src_format" ]] - then - if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] - then - src_format="zip" - elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] - then - src_format="tar.gz" - elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] - then - src_format="tar.xz" - elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] - then - src_format="tar.bz2" - elif [[ -z "$src_extract" ]] - then - src_extract="false" - fi - fi - - src_format=${src_format:-tar.gz} - src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') - src_extract=${src_extract:-true} - - if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] - then - ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" - fi - - # Gotta use this trick with 'dirname' because source_id may contain slashes x_x - mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) - src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" - - if [ "$src_format" = "docker" ]; then - src_platform="${src_platform:-"linux/$YNH_ARCH"}" - else - [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" - - # If the file was prefetched but somehow doesn't match the sum, rm and redownload it - if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status - then - rm -f "$src_filename" - fi - - # Only redownload the file if it wasnt prefetched - if [ ! -e "$src_filename" ] - then - # NB. we have to declare the var as local first, - # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work - # because local always return 0 ... - local out - # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) - out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ - || ynh_die "$out" - fi - - # Check the control sum - if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status - then - local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)" - local actual_size="$(du -hs ${src_filename} | cut --fields=1)" - rm -f ${src_filename} - ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." - fi - fi - - # Keep files to be backup/restored at the end of the helper - # Assuming $dest_dir already exists - rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ - if [ -n "$keep" ] && [ -e "$dest_dir" ]; then - local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} - mkdir -p $keep_dir - local stuff_to_keep - for stuff_to_keep in $keep; do - if [ -e "$dest_dir/$stuff_to_keep" ]; then - mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" - cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" - fi - done - fi - - if [ "$full_replace" -eq 1 ]; then - ynh_safe_rm "$dest_dir" - fi - - # Extract source into the app dir - mkdir --parents "$dest_dir" - - if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then - _ynh_apply_default_permissions $dest_dir - fi - - if [[ "$src_extract" == "false" ]]; then - if [[ -z "$src_rename" ]] - then - mv $src_filename $dest_dir - else - mv $src_filename $dest_dir/$src_rename - fi - elif [[ "$src_format" == "docker" ]]; then - "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1 - elif [[ "$src_format" == "zip" ]]; then - # Zip format - # Using of a temp directory, because unzip doesn't manage --strip-components - if $src_in_subdir; then - local tmp_dir=$(mktemp --directory) - unzip -quo $src_filename -d "$tmp_dir" - cp --archive $tmp_dir/*/. "$dest_dir" - ynh_safe_rm "$tmp_dir" - else - unzip -quo $src_filename -d "$dest_dir" - fi - ynh_safe_rm "$src_filename" - else - local strip="" - if [ "$src_in_subdir" != "false" ]; then - if [ "$src_in_subdir" == "true" ]; then - local sub_dirs=1 - else - local sub_dirs="$src_in_subdir" - fi - strip="--strip-components $sub_dirs" - fi - if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then - tar --extract --file=$src_filename --directory="$dest_dir" $strip - else - ynh_die "Archive format unrecognized." - fi - ynh_safe_rm "$src_filename" - fi - - # Apply patches - if [ -d "$YNH_APP_BASEDIR/patches/" ]; then - local patches_folder=$(realpath $YNH_APP_BASEDIR/patches/) - # Check if any file matching the pattern exists, cf https://stackoverflow.com/a/34195247 - if compgen -G "$patches_folder/${source_id}-*.patch" >/dev/null; then - pushd "$dest_dir" - for p in $patches_folder/${source_id}-*.patch; do - echo $p - patch --strip=1 <$p || ynh_print_warn "Packagers /!\\ patch $p failed to apply" - done - popd - fi - fi - - # Keep files to be backup/restored at the end of the helper - # Assuming $dest_dir already exists - if [ -n "$keep" ]; then - local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} - local stuff_to_keep - for stuff_to_keep in $keep; do - if [ -e "$keep_dir/$stuff_to_keep" ]; then - mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" - - # We add "--no-target-directory" (short option is -T) to handle the special case - # when we "keep" a folder, but then the new setup already contains the same dir (but possibly empty) - # in which case a regular "cp" will create a copy of the directory inside the directory ... - # resulting in something like /var/www/$app/data/data instead of /var/www/$app/data - # cf https://unix.stackexchange.com/q/94831 for a more elaborate explanation on the option - cp --archive --no-target-directory "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" - fi - done - fi - rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ +# usage: ynh_exec_as_app COMMAND [ARG ...] +ynh_exec_as_app() { + sudo --preserve-env=PATH -u "$app" "$@" } # Curl abstraction to help with POST requests to local pages (such as installation forms) @@ -347,8 +90,6 @@ ynh_setup_source() { # For multiple calls, cookies are persisted between each call for the same app # # `$domain` and `$path` should be defined externally (and correspond to the domain.tld and the /path (of the app?)) -# -# Requires YunoHost version 2.6.4 or higher. ynh_local_curl() { # Define url of page to curl local local_page=$(ynh_normalize_url_path $1) @@ -417,8 +158,6 @@ _acceptable_path_to_delete() { # Remove a file or a directory, checking beforehand that it's not a disastrous location to rm such as entire /var or /home # # usage: ynh_safe_rm path_to_remove -# -# Requires YunoHost version 2.6.4 or higher. ynh_safe_rm() { local target="$1" set +o xtrace # set +x @@ -445,8 +184,6 @@ ynh_safe_rm() { # usage: ynh_read_manifest "key" # | arg: key - Name of the key to find # | ret: the value associate to that key -# -# Requires YunoHost version 3.5.0 or higher. ynh_read_manifest() { cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".$1" --raw-output } @@ -457,8 +194,6 @@ ynh_read_manifest() { # | ret: the version number of the upstream app # # For example, if the manifest contains `4.3-2~ynh3` the function will return `4.3-2` -# -# Requires YunoHost version 3.5.0 or higher. ynh_app_upstream_version() { echo "${YNH_APP_MANIFEST_VERSION/~ynh*/}" } @@ -474,8 +209,6 @@ ynh_app_upstream_version_changed() { # Compare the current package version is strictly lower than another version given as an argument # # example: if ynh_app_upgrading_from_version_before 2.3.2~ynh1; then ... -# -# Requires YunoHost version 11.2 or higher. ynh_app_upgrading_from_version_before() { local version=$1 [[ $version =~ '~ynh' ]] || ynh_die "Invalid argument for version, should include the ~ynhX prefix" @@ -486,8 +219,6 @@ ynh_app_upgrading_from_version_before() { # Compare the current package version is lower or equal to another version given as an argument # # example: if ynh_app_upgrading_from_version_before_or_equal_to 2.3.2~ynh1; then ... -# -# Requires YunoHost version 11.2 or higher. ynh_app_upgrading_from_version_before_or_equal_to() { local version=$1 [[ $version =~ '~ynh' ]] || ynh_die "Invalid argument for version, should include the ~ynhX prefix" @@ -546,8 +277,6 @@ toml_to_json() { # | ret: 0 for valid ip addresses, 1 otherwise # # example: ynh_validate_ip 4 111.222.333.444 -# -# Requires YunoHost version 2.2.4 or higher. ynh_validate_ip() { # ============ Argument parsing ============= local -A args_array=([f]=family= [i]=ip_address=) @@ -576,11 +305,9 @@ EOF # [packagingv1] # # usage: ynh_get_ram [--free|--total] -# | arg: -f, --free - Count free RAM+swap -# | arg: -t, --total - Count total RAM+swap +# | arg: --free - Count free RAM+swap +# | arg: --total - Count total RAM+swap # | ret: the amount of free ram, in MB (MegaBytes) -# -# Requires YunoHost version 3.8.1 or higher. ynh_get_ram() { # ============ Argument parsing ============= local -A args_array=([f]=free [t]=total) @@ -614,8 +341,35 @@ ynh_get_ram() { # usage: ynh_in_ci_tests # # Return 0 if in CI, 1 otherwise -# -# Requires YunoHost version 11.3 or higher. ynh_in_ci_tests() { [ "${PACKAGE_CHECK_EXEC:-0}" -eq 1 ] } + +# Retrieve a YunoHost user information +# +# usage: ynh_user_get_info --username=username --key=key +# | arg: --username= - the username to retrieve info from +# | arg: --key= - the key to retrieve +# | ret: the value associate to that key +# +# example: mail=$(ynh_user_get_info --username="toto" --key=mail) +ynh_user_get_info() { + # ============ Argument parsing ============= + local -A args_array=([u]=username= [k]=key=) + local username + local key + ynh_handle_getopts_args "$@" + # =========================================== + + yunohost user info "$username" --output-as json --quiet | jq -r ".$key" +} + +# Get the list of YunoHost users +# +# usage: ynh_user_list +# | ret: one username per line as strings +# +# example: for u in $(ynh_user_list); do ... ; done +ynh_user_list() { + yunohost user list --output-as json --quiet | jq -r ".users | keys[]" +} From 0aad13cd2fcd0705db8c4a635e69416265a00bb6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 19:11:28 +0200 Subject: [PATCH 110/146] helpers2.1: oopsies in apt helper --- helpers/helpers.v2.1.d/apt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt index 00a48d64d..d8425ee58 100644 --- a/helpers/helpers.v2.1.d/apt +++ b/helpers/helpers.v2.1.d/apt @@ -222,7 +222,7 @@ EOF mkdir --parents "/etc/apt/trusted.gpg.d" # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) - wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | tee /etc/apt/trusted.gpg.d/$name.gpg >/dev/null + wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor > /etc/apt/trusted.gpg.d/$app.gpg # Update the list of package with the new repo NB: we use -o # Dir::Etc::sourcelist to only refresh this repo, because @@ -247,7 +247,7 @@ EOF ynh_safe_rm "/etc/apt/sources.list.d/$app.list" ynh_safe_rm "/etc/apt/preferences.d/$app" ynh_safe_rm "/etc/apt/trusted.gpg.d/$app.gpg" - ynh_apt_update + _ynh_apt update } # ##################### From 2af4c157d9872a4ce34a33ee6f8425643c584f93 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 21:35:35 +0200 Subject: [PATCH 111/146] helpers/mongo: less noisy output when checking the avx flag is here in /proc/cpuinfo --- helpers/helpers.v1.d/mongodb | 2 +- helpers/helpers.v2.1.d/mongodb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v1.d/mongodb b/helpers/helpers.v1.d/mongodb index d40d11bfe..8736aad31 100644 --- a/helpers/helpers.v1.d/mongodb +++ b/helpers/helpers.v1.d/mongodb @@ -310,7 +310,7 @@ ynh_install_mongo() { ynh_print_info --message="Installing MongoDB Community Edition ..." local mongo_debian_release=$(ynh_get_debian_release) - if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then + if [[ "$(grep '^flags' /proc/cpuinfo | uniq)" != *"avx"* && "$mongo_version" != "4.4" ]]; then ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." mongo_version="4.4" fi diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index da4159bc8..b0d1fe981 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -226,7 +226,7 @@ ynh_install_mongo() { ynh_print_info "Installing MongoDB Community Edition ..." local mongo_debian_release=$YNH_DEBIAN_VERSION - if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then + if [[ "$(grep '^flags' /proc/cpuinfo | uniq)" != *"avx"* && "$mongo_version" != "4.4" ]]; then ynh_print_warn "Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." mongo_version="4.4" fi From ed426f05ba3effe2ca7ef74e77969aa1a246119e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 22:13:01 +0200 Subject: [PATCH 112/146] apps/helpers2.1: fix app env in resource upgrade context ending up in incorrect helper version being used --- src/app.py | 2 +- src/utils/resources.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/app.py b/src/app.py index b729eab19..dd6acb0ef 100644 --- a/src/app.py +++ b/src/app.py @@ -770,7 +770,7 @@ def app_upgrade( from yunohost.utils.resources import AppResourceManager AppResourceManager( - app_instance_name, wanted=manifest, current=app_dict["manifest"] + app_instance_name, wanted=manifest, current=app_dict["manifest"], workdir=extracted_app_folder ).apply( rollback_and_raise_exception_if_failure=True, operation_logger=operation_logger, diff --git a/src/utils/resources.py b/src/utils/resources.py index 3e41044ea..ea6448c36 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -39,10 +39,11 @@ logger = getActionLogger("yunohost.app_resources") class AppResourceManager: - def __init__(self, app: str, current: Dict, wanted: Dict): + def __init__(self, app: str, current: Dict, wanted: Dict, workdir=None): self.app = app self.current = current self.wanted = wanted + self.workdir = workdir if "resources" not in self.current: self.current["resources"] = {} @@ -256,17 +257,17 @@ class AppResource: ) from yunohost.hook import hook_exec_with_script_debug_if_failure - tmpdir = _make_tmp_workdir_for_app(app=self.app) + workdir = self.manager.workdir if self.manager and self.manager.workdir else _make_tmp_workdir_for_app(app=self.app) env_ = _make_environment_for_app_script( self.app, - workdir=tmpdir, + workdir=workdir, action=f"{action}_{self.type}", force_include_app_settings=True, ) env_.update(env) - script_path = f"{tmpdir}/{action}_{self.type}" + script_path = f"{workdir}/{action}_{self.type}" script = f""" source /usr/share/yunohost/helpers ynh_abort_if_errors From e3bebeac98126ddde652d31f81d66da6277be12d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 22:13:16 +0200 Subject: [PATCH 113/146] helpers2.1: typo in getopts --- helpers/helpers.v2.1.d/getopts | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index d8373203d..4df68e7ae 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -51,6 +51,7 @@ ynh_handle_getopts_args() { return # Validate that the first char is - because it should be something like --option=value or -o ... elif [[ "${1:0:1}" != "-" ]] + then ynh_die "It looks like you called the helper using positional arguments instead of keyword arguments ?" fi From c9b76fde35bc1179d2ecc36ebde94874994c61cc Mon Sep 17 00:00:00 2001 From: alexAubin <4533074+alexAubin@users.noreply.github.com> Date: Mon, 24 Jun 2024 20:13:47 +0000 Subject: [PATCH 114/146] :art: Format Python code with Black --- src/app.py | 5 ++++- src/utils/resources.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index dd6acb0ef..597743cc8 100644 --- a/src/app.py +++ b/src/app.py @@ -770,7 +770,10 @@ def app_upgrade( from yunohost.utils.resources import AppResourceManager AppResourceManager( - app_instance_name, wanted=manifest, current=app_dict["manifest"], workdir=extracted_app_folder + app_instance_name, + wanted=manifest, + current=app_dict["manifest"], + workdir=extracted_app_folder, ).apply( rollback_and_raise_exception_if_failure=True, operation_logger=operation_logger, diff --git a/src/utils/resources.py b/src/utils/resources.py index ea6448c36..71c455c77 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -257,7 +257,11 @@ class AppResource: ) from yunohost.hook import hook_exec_with_script_debug_if_failure - workdir = self.manager.workdir if self.manager and self.manager.workdir else _make_tmp_workdir_for_app(app=self.app) + workdir = ( + self.manager.workdir + if self.manager and self.manager.workdir + else _make_tmp_workdir_for_app(app=self.app) + ) env_ = _make_environment_for_app_script( self.app, From 094cd9ddd675cfaf5d593b82e1922a8f64b3379b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 22:34:28 +0200 Subject: [PATCH 115/146] helpers: rework helper doc now that we have multiple versions of helpers in parallel + improve structure (group helper file in categories) --- .gitlab/ci/doc.gitlab-ci.yml | 6 ++- doc/generate_helper_doc.py | 83 +++++++++++++++++++++++++----------- doc/helper_doc_template.md | 32 ++++++++------ 3 files changed, 81 insertions(+), 40 deletions(-) diff --git a/.gitlab/ci/doc.gitlab-ci.yml b/.gitlab/ci/doc.gitlab-ci.yml index 0562275d0..1d80ffb94 100644 --- a/.gitlab/ci/doc.gitlab-ci.yml +++ b/.gitlab/ci/doc.gitlab-ci.yml @@ -12,10 +12,12 @@ generate-helpers-doc: - git config --global user.name "$GITHUB_USER" script: - cd doc - - python3 generate_helper_doc.py + - python3 generate_helper_doc.py 2 + - python3 generate_helper_doc.py 2.1 - python3 generate_resource_doc.py > resources.md - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo - - cp helpers.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md + - cp helpers.v2.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md + - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers_v2.1.md - cp resources.md doc_repo/pages/06.contribute/10.packaging_apps/10.manifest/10.appresources/packaging_app_manifest_resources.md - cd doc_repo # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ? diff --git a/doc/generate_helper_doc.py b/doc/generate_helper_doc.py index cd0b67d01..926227f3c 100644 --- a/doc/generate_helper_doc.py +++ b/doc/generate_helper_doc.py @@ -1,10 +1,40 @@ #!/usr/env/python3 +import sys import os import glob import datetime import subprocess +tree = { + "sources": { + "title": "Sources", + "notes": "This is coupled to the 'sources' resource in the manifest.toml", + "subsections": ["sources"], + }, + "tech": { + "title": "App technologies", + "notes": "These allow to install specific version of the technology required to run some apps", + "subsections": ["nodejs", "ruby", "go", "composer"], + }, + "db": { + "title": "Databases", + "notes": "This is coupled to the 'database' resource in the manifest.toml - at least for mysql/postgresql. Mongodb/redis may have better integration in the future.", + "subsections": ["mysql", "postgresql", "mongodb", "redis"], + }, + "conf": { + "title": "Configurations / templating", + "subsections": ["templating", "nginx", "php", "systemd", "fail2ban", "logrotate"], + }, + "misc": { + "title": "Misc tools", + "subsections": ["utils", "setting", "string", "backup", "logging", "multimedia"], + }, + "meh": { + "title": "Deprecated or handled by the core / app resources since v2", + "subsections": ["permission", "apt", "systemuser"], + }, +} def get_current_commit(): p = subprocess.Popen( @@ -19,14 +49,7 @@ def get_current_commit(): return current_commit -def render(helpers): - current_commit = get_current_commit() - - data = { - "helpers": helpers, - "date": datetime.datetime.now().strftime("%d/%m/%Y"), - "version": open("../debian/changelog").readlines()[0].split()[1].strip("()"), - } +def render(tree, helpers_version): from jinja2 import Template from ansi2html import Ansi2HTMLConverter @@ -42,12 +65,15 @@ def render(helpers): t = Template(template) t.globals["now"] = datetime.datetime.utcnow result = t.render( - current_commit=current_commit, - data=data, + tree=tree, + date=datetime.datetime.now().strftime("%d/%m/%Y"), + version=open("../debian/changelog").readlines()[0].split()[1].strip("()"), + helpers_version=helpers_version, + current_commit=get_current_commit(), convert=shell_to_html, shell_css=shell_css, ) - open("helpers.md", "w").write(result) + open(f"helpers.v{helpers_version}.md", "w").write(result) ############################################################################## @@ -87,7 +113,7 @@ class Parser: # We're still in a comment bloc assert line.startswith("# ") or line == "#", malformed_error(i) current_block["comments"].append(line[2:]) - elif line.strip() == "": + elif line.strip() == "" or line.startswith("_ynh"): # Well eh that was not an actual helper definition ... start over ? current_reading = "void" current_block = { @@ -121,7 +147,8 @@ class Parser: # (we ignore helpers containing [internal] ...) if ( "[packagingv1]" not in current_block["comments"] - and "[internal]" not in current_block["comments"] + and not any(line.startswith("[internal]") for line in current_block["comments"]) + and not current_block["name"].startswith("_") ): self.blocks.append(current_block) current_block = { @@ -212,23 +239,27 @@ def malformed_error(line_number): def main(): - helper_files = sorted(glob.glob("../helpers/*")) - helpers = [] - for helper_file in helper_files: - if not os.path.isfile(helper_file): - continue + if len(sys.argv) == 1: + print("This script needs the helper version (1, 2, 2.1) as an argument") + sys.exit(1) - category_name = os.path.basename(helper_file) - print("Parsing %s ..." % category_name) - p = Parser(helper_file) - p.parse_blocks() - for b in p.blocks: - p.parse_block(b) + version = sys.argv[1] - helpers.append((category_name, p.blocks)) + for section in tree.values(): + section["helpers"] = {} + for subsection in section["subsections"]: + print(f"Parsing {subsection} ...") + helper_file = f"../helpers/helpers.v{version}.d/{subsection}" + assert os.path.isfile(helper_file), f"Uhoh, {file} doesn't exists?" + p = Parser(helper_file) + p.parse_blocks() + for b in p.blocks: + p.parse_block(b) - render(helpers) + section["helpers"][subsection] = p.blocks + + render(tree, version) main() diff --git a/doc/helper_doc_template.md b/doc/helper_doc_template.md index 07d74d08c..66c53da06 100644 --- a/doc/helper_doc_template.md +++ b/doc/helper_doc_template.md @@ -1,19 +1,27 @@ --- -title: App helpers +title: App helpers (v{{ helpers_version }}) template: docs taxonomy: category: docs routes: - default: '/packaging_apps_helpers' + default: '/packaging_apps_helpers{% if helpers_version not in ["1", "2"] %}_v{{ helpers_version }}{% endif %}' --- -Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/doc/generate_helper_doc.py) on {{data.date}} (YunoHost version {{data.version}}) +Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/doc/generate_helper_doc.py) on {{date}} (YunoHost version {{version}}) -{% for category, helpers in data.helpers %} -## {{ category.upper() }} -{% for h in helpers %} -### {{ h.name }} -[details summary="{{ h.brief }}" class="helper-card-subtitle text-muted"] + +{% for section_id, section in tree.items() %} +## {{ section["title"].title() }} + +{% if section['notes'] %}

{{ section['notes'] }}

{% endif %} + + {% for subsection, helpers in section["helpers"].items() %} + +### {{ subsection.upper() }} + {% for h in helpers %} +#### {{ h.name }} + +
{{ h.brief }} **Usage**: `{{ h.usage }}` {%- if h.args %} @@ -51,9 +59,9 @@ Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ **Details**: {{ h.details }} {%- endif %} -[Dude, show me the code!](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/helpers/{{ category }}#L{{ h.line + 1 }}) -[/details] - +[Dude, show me the code!](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/helpers/helpers.v{{ helpers_version if helpers_version != "2" else "1" }}.d/{{ subsection }}#L{{ h.line + 1 }}) +
+ {% endfor %} --- -{% endfor %} + {% endfor %} {% endfor %} From 2d2693507978b04c900c6aa90210cab996f317e7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2024 22:38:18 +0200 Subject: [PATCH 116/146] Update changelog for 11.2.17 --- debian/changelog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index 1f0c6c57a..1bb5f7149 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +yunohost (11.2.17) stable; urgency=low + + - helpers: Misc cleaning / reorganizing to prepare new doc (2895d4d9) + - helpers: rework helper doc now that we have multiple versions of helpers in parallel + improve structure (group helper file in categories) (094cd9dd) + - helpers/mongo: less noisy output when checking the avx flag is here in /proc/cpuinfo (2af4c157) + - apps/helpers2.1: fix app env in resource upgrade context ending up in incorrect helper version being used (ed426f05) + - helpers2.1: forgot to propagate the 'goenv latest' fix from helpers v1 (d8c3ff4c) + - helpers2.1: drop ynh_apps helper because only a single app is using it ... (1fb80e5d) + - helpers2.1: other typo fixes + + -- Alexandre Aubin Mon, 24 Jun 2024 22:36:32 +0200 + yunohost (11.2.16) stable; urgency=low - apps/logs: fix some information not being redacted because of the packaging v2 flows (a25033bb) From 6851d740f74bcd16f49db708a08765719058e3f4 Mon Sep 17 00:00:00 2001 From: alexAubin <4533074+alexAubin@users.noreply.github.com> Date: Mon, 24 Jun 2024 20:38:46 +0000 Subject: [PATCH 117/146] :art: Format Python code with Black --- doc/generate_helper_doc.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/doc/generate_helper_doc.py b/doc/generate_helper_doc.py index 926227f3c..d572b03da 100644 --- a/doc/generate_helper_doc.py +++ b/doc/generate_helper_doc.py @@ -21,14 +21,28 @@ tree = { "title": "Databases", "notes": "This is coupled to the 'database' resource in the manifest.toml - at least for mysql/postgresql. Mongodb/redis may have better integration in the future.", "subsections": ["mysql", "postgresql", "mongodb", "redis"], - }, + }, "conf": { "title": "Configurations / templating", - "subsections": ["templating", "nginx", "php", "systemd", "fail2ban", "logrotate"], + "subsections": [ + "templating", + "nginx", + "php", + "systemd", + "fail2ban", + "logrotate", + ], }, "misc": { "title": "Misc tools", - "subsections": ["utils", "setting", "string", "backup", "logging", "multimedia"], + "subsections": [ + "utils", + "setting", + "string", + "backup", + "logging", + "multimedia", + ], }, "meh": { "title": "Deprecated or handled by the core / app resources since v2", @@ -36,6 +50,7 @@ tree = { }, } + def get_current_commit(): p = subprocess.Popen( "git rev-parse --verify HEAD", @@ -147,7 +162,10 @@ class Parser: # (we ignore helpers containing [internal] ...) if ( "[packagingv1]" not in current_block["comments"] - and not any(line.startswith("[internal]") for line in current_block["comments"]) + and not any( + line.startswith("[internal]") + for line in current_block["comments"] + ) and not current_block["name"].startswith("_") ): self.blocks.append(current_block) From 7347b08e49ae3fa091dcc96829a16310e1780763 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 25 Jun 2024 00:18:38 +0200 Subject: [PATCH 118/146] ci: Fix helpers 2.1 doc location --- .gitlab/ci/doc.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/doc.gitlab-ci.yml b/.gitlab/ci/doc.gitlab-ci.yml index 1d80ffb94..35509556a 100644 --- a/.gitlab/ci/doc.gitlab-ci.yml +++ b/.gitlab/ci/doc.gitlab-ci.yml @@ -17,7 +17,7 @@ generate-helpers-doc: - python3 generate_resource_doc.py > resources.md - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo - cp helpers.v2.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md - - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers_v2.1.md + - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/12.helpers21/packaging_app_scripts_helpers_v21.md - cp resources.md doc_repo/pages/06.contribute/10.packaging_apps/10.manifest/10.appresources/packaging_app_manifest_resources.md - cd doc_repo # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ? From 2a7fefaecb41589aa968293b41b610aa3fe14b69 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 25 Jun 2024 00:21:24 +0200 Subject: [PATCH 119/146] helpers/doc: De-hide some helpers v1 in documentation now that the structure is less bloated sort of ? --- helpers/helpers.v1.d/apt | 6 ------ helpers/helpers.v1.d/permission | 14 -------------- helpers/helpers.v1.d/systemuser | 4 ---- 3 files changed, 24 deletions(-) diff --git a/helpers/helpers.v1.d/apt b/helpers/helpers.v1.d/apt index d5a1f4335..8231515fc 100644 --- a/helpers/helpers.v1.d/apt +++ b/helpers/helpers.v1.d/apt @@ -224,8 +224,6 @@ YNH_INSTALL_APP_DEPENDENCIES_REPLACE="true" # Define and install dependencies with a equivs control file # -# [packagingv1] -# # This helper can/should only be called once per app # # example : ynh_install_app_dependencies dep1 dep2 "dep3|dep4|dep5" @@ -364,8 +362,6 @@ ynh_add_app_dependencies() { # Remove fake package and its dependencies # -# [packagingv1] -# # Dependencies will removed only if no other package need them. # # usage: ynh_remove_app_dependencies @@ -398,8 +394,6 @@ ynh_remove_app_dependencies() { # Install packages from an extra repository properly. # -# [packagingv1] -# # usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" [--key=key_url] [--name=name] # | arg: -r, --repo= - Complete url of the extra repository. # | arg: -p, --package= - The packages to install from this extra repository diff --git a/helpers/helpers.v1.d/permission b/helpers/helpers.v1.d/permission index d3eb71c22..6c2fa7ef8 100644 --- a/helpers/helpers.v1.d/permission +++ b/helpers/helpers.v1.d/permission @@ -36,8 +36,6 @@ # | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. Defaults to false for the permission different than 'main'. # | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. # -# [packagingv1] -# # If provided, 'url' or 'additional_urls' is assumed to be relative to the app domain/path if they # start with '/'. For example: # / -> domain.tld/app @@ -145,8 +143,6 @@ ynh_permission_create() { # Remove a permission for the app (note that when the app is removed all permission is automatically removed) # -# [packagingv1] -# # example: ynh_permission_delete --permission=editors # # usage: ynh_permission_delete --permission="permission" @@ -165,8 +161,6 @@ ynh_permission_delete() { # Check if a permission exists # -# [packagingv1] -# # usage: ynh_permission_exists --permission=permission # | arg: -p, --permission= - the permission to check # | exit: Return 1 if the permission doesn't exist, 0 otherwise @@ -185,8 +179,6 @@ ynh_permission_exists() { # Redefine the url associated to a permission # -# [packagingv1] -# # usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]] # [--auth_header=true|false] [--clear_urls] # | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) @@ -255,8 +247,6 @@ ynh_permission_url() { # Update a permission for the app # -# [packagingv1] -# # usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]] # [--label="label"] [--show_tile=true|false] [--protected=true|false] # | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) @@ -362,8 +352,6 @@ ynh_permission_has_user() { # Check if a legacy permissions exist # -# [packagingv1] -# # usage: ynh_legacy_permissions_exists # | exit: Return 1 if the permission doesn't exist, 0 otherwise # @@ -379,8 +367,6 @@ ynh_legacy_permissions_exists() { # Remove all legacy permissions # -# [packagingv1] -# # usage: ynh_legacy_permissions_delete_all # # example: diff --git a/helpers/helpers.v1.d/systemuser b/helpers/helpers.v1.d/systemuser index 064cae4ae..4fac34bb4 100644 --- a/helpers/helpers.v1.d/systemuser +++ b/helpers/helpers.v1.d/systemuser @@ -42,8 +42,6 @@ ynh_system_group_exists() { # Create a system user # -# [packagingv1] -# # usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell] [--groups="group1 group2"] # | arg: -u, --username= - Name of the system user that will be create # | arg: -h, --home_dir= - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home @@ -98,8 +96,6 @@ ynh_system_user_create() { # Delete a system user # -# [packagingv1] -# # usage: ynh_system_user_delete --username=user_name # | arg: -u, --username= - Name of the system user that will be create # From 997388dc7959e3046fefd3f3308802f9d3618868 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 25 Jun 2024 14:15:51 +0200 Subject: [PATCH 120/146] helpers2.1: fix __PATH__/ handling --- helpers/helpers.v2.1.d/templating | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/templating b/helpers/helpers.v2.1.d/templating index ffd161278..dc7a090a2 100644 --- a/helpers/helpers.v2.1.d/templating +++ b/helpers/helpers.v2.1.d/templating @@ -111,7 +111,7 @@ _ynh_replace_vars() { # Specific trick to make sure that __PATH__/ doesn't end up in "//" if $path=/ if [[ "${path:-}" == "/" ]] && grep -q '__PATH__/' $file; then - sed --in-place "s@__PATH__/@${path%/}@g" "$file" + sed --in-place "s@__PATH__/@/@g" "$file" fi # Do the replacement From feb9a095b3c4435701079b7ac1c5c943c9173764 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 25 Jun 2024 14:17:49 +0200 Subject: [PATCH 121/146] helpers doc: fix detail block, cant use the HTML
because grav doesnt interpret markdown in it --- doc/helper_doc_template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/helper_doc_template.md b/doc/helper_doc_template.md index 66c53da06..53bec79fc 100644 --- a/doc/helper_doc_template.md +++ b/doc/helper_doc_template.md @@ -21,7 +21,7 @@ Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ {% for h in helpers %} #### {{ h.name }} -
{{ h.brief }} +[details summary="{{ h.brief }}" class="helper-card-subtitle text-muted"] **Usage**: `{{ h.usage }}` {%- if h.args %} @@ -60,7 +60,7 @@ Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ {{ h.details }} {%- endif %} [Dude, show me the code!](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/helpers/helpers.v{{ helpers_version if helpers_version != "2" else "1" }}.d/{{ subsection }}#L{{ h.line + 1 }}) -
+[/details] {% endfor %} --- {% endfor %} From 87eedc2a369baa72c9b8d46154924f667885c085 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 25 Jun 2024 14:20:42 +0200 Subject: [PATCH 122/146] Update changelog for 11.2.17.1 --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index 1bb5f7149..2ea7acce1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +yunohost (11.2.17.1) stable; urgency=low + + - helpers2.1: fix __PATH__/ handling (997388dc) + - ci: Fix helpers 2.1 doc location (7347b08e) + - helpers/doc: De-hide some helpers v1 in documentation now that the structure is less bloated sort of ? (2a7fefae) + - helpers/doc: fix detail block, cant use the HTML
because grav doesnt interpret markdown in it (feb9a095) + + -- Alexandre Aubin Tue, 25 Jun 2024 14:19:58 +0200 + yunohost (11.2.17) stable; urgency=low - helpers: Misc cleaning / reorganizing to prepare new doc (2895d4d9) From 650481a58ae1663aa44ecb805f70cafde469859a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 28 Sep 2023 11:23:59 +0200 Subject: [PATCH 123/146] ynh_safe_rm: Check if target is a symlink When calling ynh_safe_rm to a broken symlink, the function was erroring out. (test -e was following the symlink and returning false) We need to also check if it is a symlink before exiting. --- helpers/helpers.v2.1.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index be57a6d73..0cbd7c004 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -168,7 +168,7 @@ ynh_safe_rm() { if [[ -z "$target" ]]; then ynh_print_warn "ynh_safe_rm called with empty argument, ignoring." - elif [[ ! -e $target ]]; then + elif [[ ! -e "$target" ]] && [[ ! -L "$target" ]]; then ynh_print_info "'$target' wasn't deleted because it doesn't exist." elif ! _acceptable_path_to_delete "$target"; then ynh_print_warn "Not deleting '$target' because it is not an acceptable path to delete." From 8846381d4758dd891ba7cc218778851669069109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 28 Jun 2024 14:43:46 +0200 Subject: [PATCH 124/146] Rework _ynh_apply_default_permissions, only check if target is a child of install_dir. --- helpers/helpers.v2.1.d/utils | 47 ++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index be57a6d73..ff378a951 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -226,41 +226,36 @@ ynh_app_upgrading_from_version_before_or_equal_to() { dpkg --compare-versions $YNH_APP_CURRENT_VERSION le $version } -# Check if we should enforce sane default permissions (= disable rwx for 'others') -# on file/folders handled with ynh_setup_source and ynh_config_add +# Apply sane permissions for files installed by ynh_setup_source and ynh_config_add. # # [internal] # -# Having a file others-readable or a folder others-executable(=enterable) -# is a security risk comparable to "chmod 777" -# -# Configuration files may contain secrets. Or even just being able to enter a -# folder may allow an attacker to do nasty stuff (maybe a file or subfolder has -# some write permission enabled for 'other' and the attacker may edit the -# content or create files as leverage for priviledge escalation ...) -# -# The sane default should be to set ownership to $app:$app. -# In specific case, you may want to set the ownership to $app:www-data -# for example if nginx needs access to static files. +# * Anything below $install_dir is chown $app:$app and chmod o-rwx,g-w +# * The rest is considered as system configuration and chown root, chmod 400 # _ynh_apply_default_permissions() { local target=$1 - chmod o-rwx $target - chmod g-w $target - chown -R root:root $target - if ynh_system_user_exists --username=$app; then - chown $app:$app $target + is_subdir() { + # Returns false if child or parent is empty + child=$(realpath "$1" 2>/dev/null) + parent=$(realpath "$2" 2>/dev/null) + [[ "${child/$parent/}" != "$child" ]] + } + + # App files can have files of their own + if ynh_system_user_exists --username="$app"; then + if is_subdir "$target" "$install_dir" || is_subdir "$target" "$data_dir"; then + chmod -R u=rwX,g=rX,o=X "$target" + chown -R "$app:$app" "$target" + chown "$app:www-data" "$target" + return + fi fi - # Crons should be owned by root - # Also we don't want systemd conf, nginx conf or others stuff to be owned by the app, - # otherwise they could self-edit their own systemd conf and escalate privilege - if echo "$target" | grep -q '^/etc/cron\|/etc/php\|/etc/nginx/conf.d\|/etc/fail2ban\|/etc/systemd/system' - then - chmod 400 $target - chown root:root $target - fi + # Other files are considered system + chmod -R 400 "$target" + chown -R root:root "$target" } int_to_bool() { From d9d404a5b24f1745f414a1bce47fda3c44007d1e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 16:06:40 +0200 Subject: [PATCH 125/146] ynh_setup_source: apply default perms *after* extracting files to hopefully remove the need to manually chown/chmod --- helpers/helpers.v2.1.d/sources | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helpers/helpers.v2.1.d/sources b/helpers/helpers.v2.1.d/sources index ced0e05c7..c0c2fb863 100644 --- a/helpers/helpers.v2.1.d/sources +++ b/helpers/helpers.v2.1.d/sources @@ -182,10 +182,6 @@ ynh_setup_source() { # Extract source into the app dir mkdir --parents "$dest_dir" - if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then - _ynh_apply_default_permissions $dest_dir - fi - if [[ "$src_extract" == "false" ]]; then if [[ -z "$src_rename" ]] then @@ -258,4 +254,8 @@ ynh_setup_source() { done fi rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ + + if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then + _ynh_apply_default_permissions $dest_dir + fi } From 3608c5678c145ce4b02caca350c913ef67a1a41f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 16:45:43 +0200 Subject: [PATCH 126/146] Proper 'if' cases to distinguish between $install_dir vs regular files in $install_dir and $data_dir --- helpers/helpers.v2.1.d/utils | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index ff378a951..6c63fe82d 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -236,7 +236,7 @@ ynh_app_upgrading_from_version_before_or_equal_to() { _ynh_apply_default_permissions() { local target=$1 - is_subdir() { + is_in_dir() { # Returns false if child or parent is empty child=$(realpath "$1" 2>/dev/null) parent=$(realpath "$2" 2>/dev/null) @@ -245,17 +245,27 @@ _ynh_apply_default_permissions() { # App files can have files of their own if ynh_system_user_exists --username="$app"; then - if is_subdir "$target" "$install_dir" || is_subdir "$target" "$data_dir"; then - chmod -R u=rwX,g=rX,o=X "$target" - chown -R "$app:$app" "$target" - chown "$app:www-data" "$target" + # If this is a file in $install_dir or $data_dir : it should be owned and read+writable by $app only + if [ -f "$target" ] && (([[ -z "${install_dir:-}" ]] is_in_dir "$target" "$install_dir") || ([[ -z "${install_dir:-}" ]] is_in_dir "$target" "$data_dir")) + then + chmod 600 "$target" + chown "$app:$app" "$target" + return + fi + # If this is the install dir (so far this is the only way this helper is called with a directory) + if [ "$target" == "$install_dir" ] + then + # Files inside should be owned by $app/www-data with rw-r----- (+x for folders or files that already have +x) + chmod -R u=rwX,g=r-X,o=--- "$target" + # We set the group to www-data because most apps do serve static assets that need to be readable by nginx ... + chown -R "$app:www-data" "$target" return fi fi # Other files are considered system - chmod -R 400 "$target" - chown -R root:root "$target" + chmod 400 "$target" + chown root:root "$target" } int_to_bool() { From 3b26ccc2a542c67769af5b7150137500dd1eb26f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 16:55:39 +0200 Subject: [PATCH 127/146] Properly handle case where $parent is empty to simplify condition --- helpers/helpers.v2.1.d/utils | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 6c63fe82d..af2859d78 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -237,23 +237,24 @@ _ynh_apply_default_permissions() { local target=$1 is_in_dir() { - # Returns false if child or parent is empty - child=$(realpath "$1" 2>/dev/null) - parent=$(realpath "$2" 2>/dev/null) + # Returns false if parent is empty + [ -n "$2" ] || return 1 + local child=$(realpath "$1" 2>/dev/null) + local parent=$(realpath "$2" 2>/dev/null) [[ "${child/$parent/}" != "$child" ]] } # App files can have files of their own if ynh_system_user_exists --username="$app"; then # If this is a file in $install_dir or $data_dir : it should be owned and read+writable by $app only - if [ -f "$target" ] && (([[ -z "${install_dir:-}" ]] is_in_dir "$target" "$install_dir") || ([[ -z "${install_dir:-}" ]] is_in_dir "$target" "$data_dir")) + if [ -f "$target" ] && (is_in_dir "$target" "${install_dir:-}" || is_in_dir "$target" "${data_dir:-}") then chmod 600 "$target" chown "$app:$app" "$target" return fi # If this is the install dir (so far this is the only way this helper is called with a directory) - if [ "$target" == "$install_dir" ] + if [ "$target" == "${install_dir:-}" ] then # Files inside should be owned by $app/www-data with rw-r----- (+x for folders or files that already have +x) chmod -R u=rwX,g=r-X,o=--- "$target" From 75d704297470d358af7d6864df76f050f5fda7b8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin <4533074+alexAubin@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:56:28 +0200 Subject: [PATCH 128/146] Update helpers/helpers.v2.1.d/utils: use regex matching to check if path is child from a parent path --- helpers/helpers.v2.1.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index af2859d78..2ee0b5d38 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -241,7 +241,7 @@ _ynh_apply_default_permissions() { [ -n "$2" ] || return 1 local child=$(realpath "$1" 2>/dev/null) local parent=$(realpath "$2" 2>/dev/null) - [[ "${child/$parent/}" != "$child" ]] + [[ "${child}" =~ ^$parent ]] } # App files can have files of their own From 8b8768fd776f9d15efd5aeb9d65cd943c0bfc920 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 18:09:35 +0200 Subject: [PATCH 129/146] Only set www-data as group for webapps --- helpers/helpers.v2.1.d/utils | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 2ee0b5d38..f03e67804 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -258,8 +258,13 @@ _ynh_apply_default_permissions() { then # Files inside should be owned by $app/www-data with rw-r----- (+x for folders or files that already have +x) chmod -R u=rwX,g=r-X,o=--- "$target" + local group="$app" # We set the group to www-data because most apps do serve static assets that need to be readable by nginx ... - chown -R "$app:www-data" "$target" + # The fact that the app is a webapp is infered by the fact that $domain and $path are defined + if [[ -n "${domain:-}" ]] && [[ -n "${path:-}" ]] then + group="www-data" + fi + chown -R "$app:$group" "$target" return fi fi From ae3018cdd0d97f4f56117fdc80bacb9637e5426e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 18:39:12 +0200 Subject: [PATCH 130/146] Infer the necessity to use www-data as group from the presence of alias or root in nginx.conf --- helpers/helpers.v2.1.d/utils | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index f03e67804..05a859498 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -256,14 +256,15 @@ _ynh_apply_default_permissions() { # If this is the install dir (so far this is the only way this helper is called with a directory) if [ "$target" == "${install_dir:-}" ] then - # Files inside should be owned by $app/www-data with rw-r----- (+x for folders or files that already have +x) - chmod -R u=rwX,g=r-X,o=--- "$target" local group="$app" - # We set the group to www-data because most apps do serve static assets that need to be readable by nginx ... - # The fact that the app is a webapp is infered by the fact that $domain and $path are defined - if [[ -n "${domain:-}" ]] && [[ -n "${path:-}" ]] then + # We set the group to www-data for webapps that do serve static assets, which therefore need to be readable by nginx ... + # The fact that the app needs this is infered by the existence of an nginx.conf and the presence of "alias" or "root" directive + if grep -q '^\s*alias\s\|^\s*root\s' "$YNH_APP_BASEDIR/conf/nginx.conf" 2>/dev/null then group="www-data" fi + # Files inside should be owned by $app with rw-r----- (+x for folders or files that already have +x) + # The group needs read/dirtraversal (in particular if it's www-data) + chmod -R u=rwX,g=r-X,o=--- "$target" chown -R "$app:$group" "$target" return fi From 656ff823a914a15741be2e756d3f5dbcdb893184 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 18:56:18 +0200 Subject: [PATCH 131/146] Also handle files in /etc/$app --- helpers/helpers.v2.1.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 05a859498..73b32a3f3 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -247,7 +247,7 @@ _ynh_apply_default_permissions() { # App files can have files of their own if ynh_system_user_exists --username="$app"; then # If this is a file in $install_dir or $data_dir : it should be owned and read+writable by $app only - if [ -f "$target" ] && (is_in_dir "$target" "${install_dir:-}" || is_in_dir "$target" "${data_dir:-}") + if [ -f "$target" ] && (is_in_dir "$target" "${install_dir:-}" || is_in_dir "$target" "${data_dir:-}" || is_in_dir "$target" "/etc/$app") then chmod 600 "$target" chown "$app:$app" "$target" From ef68485c5fc4fea221e13e05b6ed638ab519dc2f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 19:24:07 +0200 Subject: [PATCH 132/146] Use the group defined in the manifest by default --- helpers/helpers.v2.1.d/utils | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 73b32a3f3..ad7a7620d 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -256,11 +256,19 @@ _ynh_apply_default_permissions() { # If this is the install dir (so far this is the only way this helper is called with a directory) if [ "$target" == "${install_dir:-}" ] then - local group="$app" - # We set the group to www-data for webapps that do serve static assets, which therefore need to be readable by nginx ... - # The fact that the app needs this is infered by the existence of an nginx.conf and the presence of "alias" or "root" directive - if grep -q '^\s*alias\s\|^\s*root\s' "$YNH_APP_BASEDIR/conf/nginx.conf" 2>/dev/null then - group="www-data" + # Read the group from the install_dir manifest resource + local group="$(ynh_read_manifest '.resources.install_dir.group' | sed 's/null//g' | sed "s/__APP__/$app/g" | cut -f1 -d:)" + if [[ -z "$group" ]] + then + # We set the group to www-data for webapps that do serve static assets, which therefore need to be readable by nginx ... + # The fact that the app needs this is infered by the existence of an nginx.conf and the presence of "alias" or "root" directive + if grep -q '^\s*alias\s\|^\s*root\s' "$YNH_APP_BASEDIR/conf/nginx.conf" 2>/dev/null; + then + group="www-data" + # Or default to "$app" + else + group="$app" + fi fi # Files inside should be owned by $app with rw-r----- (+x for folders or files that already have +x) # The group needs read/dirtraversal (in particular if it's www-data) From 1dfc47d1d7784a355b8ba8e7213313efaacb813a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jun 2024 20:21:56 +0200 Subject: [PATCH 133/146] helpers2.1: in logrotate, make sure to also chown $app the log dir --- helpers/helpers.v2.1.d/logrotate | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate index 07e62261f..4f2d063b1 100644 --- a/helpers/helpers.v2.1.d/logrotate +++ b/helpers/helpers.v2.1.d/logrotate @@ -24,9 +24,11 @@ ynh_config_add_logrotate() { for stuff in $logfile do - mkdir --parents $(dirname "$stuff") # Make sure the permissions of the parent dir are correct (otherwise the config file could be ignored and the corresponding logs never rotated) - chmod 750 $(dirname "$stuff") + local dir=$(dirname "$stuff") + mkdir --parents $dir + chmod 750 $dir + chown $app:$app $dir done local tempconf="$(mktemp)" From 7b2959a3ebd76c50cd87287c0d3f36cd3c7345a0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 17:18:20 +0200 Subject: [PATCH 134/146] helpers2.1: forgot to rename the apt call in mongodb helpers --- helpers/helpers.v2.1.d/mongodb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb index b0d1fe981..3fb851277 100644 --- a/helpers/helpers.v2.1.d/mongodb +++ b/helpers/helpers.v2.1.d/mongodb @@ -235,7 +235,10 @@ ynh_install_mongo() { mongo_debian_release=buster fi - ynh_install_extra_app_dependencies --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc" + ynh_apt_install_dependencies_from_extra_repository \ + --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" \ + --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" \ + --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc" mongodb_servicename=mongod # Make sure MongoDB is started and enabled From 1ab3a79d3951ce9f062110361b9643495a90ac86 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 18:06:40 +0200 Subject: [PATCH 135/146] Update changelog for 11.2.18 --- debian/changelog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/debian/changelog b/debian/changelog index 2ea7acce1..f3d38c9c7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +yunohost (11.2.18) stable; urgency=low + + - helpers2.1: Rework _ynh_apply_default_permissions to hopefully remove the necessity to chown/chmod in the app scripts ([#1883](http://github.com/YunoHost/yunohost/pull/1883)) + - helpers2.1: in logrotate, make sure to also chown $app the log dir (1dfc47d1d) + - helpers2.1: forgot to rename the apt call in mongodb helpers (7b2959a3e) + - helpers2.1: in ynh_safe_rm, check if target is not a broken symlink before erorring out ([#1716](http://github.com/YunoHost/yunohost/pull/1716)) + + Thanks to all contributors <3 ! (Félix Piédallu) + + -- Alexandre Aubin Sat, 29 Jun 2024 18:05:04 +0200 + yunohost (11.2.17.1) stable; urgency=low - helpers2.1: fix __PATH__/ handling (997388dc) From 3e1c9ebaf7eca7f8ba59da0eb02cd71782b4e45d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 19:21:08 +0200 Subject: [PATCH 136/146] Fix getopts error handling ... --- helpers/helpers.v2.1.d/getopts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts index 4df68e7ae..ca517eebd 100644 --- a/helpers/helpers.v2.1.d/getopts +++ b/helpers/helpers.v2.1.d/getopts @@ -102,9 +102,9 @@ ynh_handle_getopts_args() { getopts ":$getopts_parameters" parameter || true if [ "$parameter" = "?" ]; then - ynh_die "Invalid argument: -${OPTARG:-}" + ynh_die "Invalid argument: ${1:-}" elif [ "$parameter" = ":" ]; then - ynh_die "-$OPTARG parameter requires an argument." + ynh_die "${1:-} parameter requires an argument." else local shift_value=1 # Use the long option, corresponding to the short option read by getopts, as a variable From a349fc0334d7c40a35dfc9dc7ecc8723e5fc8edd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 20:04:19 +0200 Subject: [PATCH 137/146] apps: tweaks to be more robust and prevent the stupid flood of 'sh: 0: getcwd() failed: No such file or directory' when running an app upgrade/remove from /var/www/$app, sometimes making it look like the upgrade failed when it didnt --- src/service.py | 6 ++++-- src/utils/system.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/service.py b/src/service.py index 5e49dfc8a..977b9a902 100644 --- a/src/service.py +++ b/src/service.py @@ -688,13 +688,15 @@ def _get_services(): ] for name in services_with_package_condition: package = services[name]["ignore_if_package_is_not_installed"] - if os.system(f"dpkg --list | grep -q 'ii *{package}'") != 0: + if check_output(f"dpkg-query --show --showformat='${{db:Status-Status}}' '{package}' 2>/dev/null || true") != "installed": del services[name] php_fpm_versions = check_output( - r"dpkg --list | grep -P 'ii php\d.\d-fpm' | awk '{print $2}' | grep -o -P '\d.\d' || true" + r"dpkg --list | grep -P 'ii php\d.\d-fpm' | awk '{print $2}' | grep -o -P '\d.\d' || true", + cwd="/tmp" ) php_fpm_versions = [v for v in php_fpm_versions.split("\n") if v.strip()] + for version in php_fpm_versions: # Skip php 7.3 which is most likely dead after buster->bullseye migration # because users get spooked diff --git a/src/utils/system.py b/src/utils/system.py index 27ef98dd1..5bea7f971 100644 --- a/src/utils/system.py +++ b/src/utils/system.py @@ -159,7 +159,7 @@ def ynh_packages_version(*args, **kwargs): def dpkg_is_broken(): - if check_output("dpkg --audit") != "": + if check_output("dpkg --audit", cwd="/tmp/") != "": return True # If dpkg is broken, /var/lib/dpkg/updates # will contains files like 0001, 0002, ... From c2d69f7f84c0a164b9c03df1852ba854c8aa6181 Mon Sep 17 00:00:00 2001 From: alexAubin <4533074+alexAubin@users.noreply.github.com> Date: Sat, 29 Jun 2024 18:05:02 +0000 Subject: [PATCH 138/146] :art: Format Python code with Black --- src/service.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/service.py b/src/service.py index 977b9a902..b0dc82827 100644 --- a/src/service.py +++ b/src/service.py @@ -688,12 +688,17 @@ def _get_services(): ] for name in services_with_package_condition: package = services[name]["ignore_if_package_is_not_installed"] - if check_output(f"dpkg-query --show --showformat='${{db:Status-Status}}' '{package}' 2>/dev/null || true") != "installed": + if ( + check_output( + f"dpkg-query --show --showformat='${{db:Status-Status}}' '{package}' 2>/dev/null || true" + ) + != "installed" + ): del services[name] php_fpm_versions = check_output( r"dpkg --list | grep -P 'ii php\d.\d-fpm' | awk '{print $2}' | grep -o -P '\d.\d' || true", - cwd="/tmp" + cwd="/tmp", ) php_fpm_versions = [v for v in php_fpm_versions.split("\n") if v.strip()] From d47c87e57d0bf7714416ac2ce07b400cb527a179 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 20:08:14 +0200 Subject: [PATCH 139/146] helpers2.1: wrmbgl --- helpers/helpers.v2.1.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils index 00c67c792..859cd29d9 100644 --- a/helpers/helpers.v2.1.d/utils +++ b/helpers/helpers.v2.1.d/utils @@ -257,7 +257,7 @@ _ynh_apply_default_permissions() { if [ "$target" == "${install_dir:-}" ] then # Read the group from the install_dir manifest resource - local group="$(ynh_read_manifest '.resources.install_dir.group' | sed 's/null//g' | sed "s/__APP__/$app/g" | cut -f1 -d:)" + local group="$(ynh_read_manifest 'resources.install_dir.group' | sed 's/null//g' | sed "s/__APP__/$app/g" | cut -f1 -d:)" if [[ -z "$group" ]] then # We set the group to www-data for webapps that do serve static assets, which therefore need to be readable by nginx ... From e5b575901a77b2fea13d3f81aed64366566efc8c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 20:31:28 +0200 Subject: [PATCH 140/146] apps: be more robust when an app upgrade succeeds but for some reason is marked with 'broke the system' ... ending up in inconsistent state between the app settings vs the app scritpts (for example in v1->v2 transitions but not only) --- src/app.py | 63 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/src/app.py b/src/app.py index 597743cc8..f01630d71 100644 --- a/src/app.py +++ b/src/app.py @@ -849,6 +849,39 @@ def app_upgrade( + "\n -".join(manually_modified_files_by_app) ) + # If the upgrade didnt fail, update the revision and app files (even if it broke the system, otherwise we end up in a funky intermediate state where the app files don't match the installed version or settings, for example for v1->v2 upgrade marked as "broke the system" for some reason) + if not upgrade_failed: + now = int(time.time()) + app_setting(app_instance_name, "update_time", now) + app_setting( + app_instance_name, + "current_revision", + manifest.get("remote", {}).get("revision", "?"), + ) + + # Clean hooks and add new ones + hook_remove(app_instance_name) + if "hooks" in os.listdir(extracted_app_folder): + for hook in os.listdir(extracted_app_folder + "/hooks"): + hook_add(app_instance_name, extracted_app_folder + "/hooks/" + hook) + + # Replace scripts and manifest and conf (if exists) + # Move scripts and manifest to the right place + for file_to_copy in APP_FILES_TO_COPY: + rm(f"{app_setting_path}/{file_to_copy}", recursive=True, force=True) + if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): + cp( + f"{extracted_app_folder}/{file_to_copy}", + f"{app_setting_path}/{file_to_copy}", + recursive=True, + ) + + # Clean and set permissions + shutil.rmtree(extracted_app_folder) + chmod(app_setting_path, 0o600) + chmod(f"{app_setting_path}/settings.yml", 0o400) + chown(app_setting_path, "root", recursive=True) + # If upgrade failed or broke the system, # raise an error and interrupt all other pending upgrades if upgrade_failed or broke_the_system: @@ -899,36 +932,6 @@ def app_upgrade( ) # Otherwise we're good and keep going ! - now = int(time.time()) - app_setting(app_instance_name, "update_time", now) - app_setting( - app_instance_name, - "current_revision", - manifest.get("remote", {}).get("revision", "?"), - ) - - # Clean hooks and add new ones - hook_remove(app_instance_name) - if "hooks" in os.listdir(extracted_app_folder): - for hook in os.listdir(extracted_app_folder + "/hooks"): - hook_add(app_instance_name, extracted_app_folder + "/hooks/" + hook) - - # Replace scripts and manifest and conf (if exists) - # Move scripts and manifest to the right place - for file_to_copy in APP_FILES_TO_COPY: - rm(f"{app_setting_path}/{file_to_copy}", recursive=True, force=True) - if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): - cp( - f"{extracted_app_folder}/{file_to_copy}", - f"{app_setting_path}/{file_to_copy}", - recursive=True, - ) - - # Clean and set permissions - shutil.rmtree(extracted_app_folder) - chmod(app_setting_path, 0o600) - chmod(f"{app_setting_path}/settings.yml", 0o400) - chown(app_setting_path, "root", recursive=True) # So much win logger.success(m18n.n("app_upgraded", app=app_instance_name)) From dbf579b7b4803e3f5478bd0dd3870dd92f10f8a6 Mon Sep 17 00:00:00 2001 From: alexAubin <4533074+alexAubin@users.noreply.github.com> Date: Sat, 29 Jun 2024 18:31:51 +0000 Subject: [PATCH 141/146] :art: Format Python code with Black --- src/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index f01630d71..f7e17a269 100644 --- a/src/app.py +++ b/src/app.py @@ -863,7 +863,9 @@ def app_upgrade( hook_remove(app_instance_name) if "hooks" in os.listdir(extracted_app_folder): for hook in os.listdir(extracted_app_folder + "/hooks"): - hook_add(app_instance_name, extracted_app_folder + "/hooks/" + hook) + hook_add( + app_instance_name, extracted_app_folder + "/hooks/" + hook + ) # Replace scripts and manifest and conf (if exists) # Move scripts and manifest to the right place From ff78f3ada7c118869a7b1d8dfc07e1b6894dcc83 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sat, 29 Jun 2024 20:57:21 +0200 Subject: [PATCH 142/146] automatically ignore the service in diagnosis if it has been deactivated with the ynh cli --- src/service.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/service.py b/src/service.py index b0dc82827..0a0c37378 100644 --- a/src/service.py +++ b/src/service.py @@ -26,6 +26,7 @@ from glob import glob from datetime import datetime from moulinette import m18n +from yunohost.utils.diagnosis import diagnosis_ignore, diagnosis_unignore from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger @@ -296,6 +297,9 @@ def service_enable(names): names = [names] for name in names: if _run_service_command("enable", name): + services = _get_services() + if name in services: + diagnosis_unignore({"services": [{"service": name}]}) logger.success(m18n.n("service_enabled", service=name)) else: raise YunohostError( @@ -315,6 +319,9 @@ def service_disable(names): names = [names] for name in names: if _run_service_command("disable", name): + services = _get_services() + if name in services: + diagnosis_ignore({"services": [{"service": name}]}) logger.success(m18n.n("service_disabled", service=name)) else: raise YunohostError( From eaf00103dd72c01564f245f85f8e0ee68662c3ac Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sat, 29 Jun 2024 20:57:59 +0200 Subject: [PATCH 143/146] Revert "automatically ignore the service in diagnosis if it has been deactivated with the ynh cli" This reverts commit ff78f3ada7c118869a7b1d8dfc07e1b6894dcc83. --- src/service.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/service.py b/src/service.py index 0a0c37378..b0dc82827 100644 --- a/src/service.py +++ b/src/service.py @@ -26,7 +26,6 @@ from glob import glob from datetime import datetime from moulinette import m18n -from yunohost.utils.diagnosis import diagnosis_ignore, diagnosis_unignore from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger @@ -297,9 +296,6 @@ def service_enable(names): names = [names] for name in names: if _run_service_command("enable", name): - services = _get_services() - if name in services: - diagnosis_unignore({"services": [{"service": name}]}) logger.success(m18n.n("service_enabled", service=name)) else: raise YunohostError( @@ -319,9 +315,6 @@ def service_disable(names): names = [names] for name in names: if _run_service_command("disable", name): - services = _get_services() - if name in services: - diagnosis_ignore({"services": [{"service": name}]}) logger.success(m18n.n("service_disabled", service=name)) else: raise YunohostError( From eee84c5f6670d4e9be1c59516696e3d76347246a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 21:32:53 +0200 Subject: [PATCH 144/146] helpers2.1: also run _ynh_apply_default_permissions in ynh_restore to be consistent (also because the user uid on the new system may be different than in the archive etc) --- helpers/helpers.v2.1.d/backup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup index 0668d3e17..a40c4f1f2 100644 --- a/helpers/helpers.v2.1.d/backup +++ b/helpers/helpers.v2.1.d/backup @@ -179,6 +179,8 @@ ynh_restore() { else mv "$archive_path" "${target}" fi + + _ynh_apply_default_permissions "$target" } # Restore all files that were previously backuped in an app backup script From c2271ab7310025affcbd73ba33c340ad92ec7712 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jun 2024 23:57:21 +0200 Subject: [PATCH 145/146] Update changelog for 11.2.19 --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index f3d38c9c7..7a8dfacc9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +yunohost (11.2.19) stable; urgency=low + + - apps: tweaks to be more robust and prevent the stupid flood of 'sh: 0: getcwd() failed: No such file or directory' when running an app upgrade/remove from /var/www/$app, sometimes making it look like the upgrade failed when it didnt (a349fc03) + - apps: be more robust when an app upgrade succeeds but for some reason is marked with 'broke the system' ... ending up in inconsistent state between the app settings vs the app scritpts (for example in v1->v2 transitions but not only) (e5b57590) + - helpers2.1: Fix getopts error handling ... (3e1c9eba) + - helpers2.1: also run _ynh_apply_default_permissions in ynh_restore to be consistent (also because the user uid on the new system may be different than in the archive etc) (eee84c5f) + + -- Alexandre Aubin Sat, 29 Jun 2024 23:55:52 +0200 + yunohost (11.2.18) stable; urgency=low - helpers2.1: Rework _ynh_apply_default_permissions to hopefully remove the necessity to chown/chmod in the app scripts ([#1883](http://github.com/YunoHost/yunohost/pull/1883)) From a18d5f26f2b42191261d6ab3755ff838d69f8aa0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 30 Jun 2024 00:21:40 +0200 Subject: [PATCH 146/146] helpers2.1: zgrblg --- helpers/helpers.v2.1.d/go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go index c0b8e9022..bb272d50c 100644 --- a/helpers/helpers.v2.1.d/go +++ b/helpers/helpers.v2.1.d/go @@ -97,7 +97,7 @@ ynh_go_install () { test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go # Install the requested version of Go - local final_go_version=$("$goenv_latest_dir/bin/goenv-latest" --print "$go_version") + local final_go_version=$("$GOENV_INSTALL_DIR/plugins/xxenv-latest/bin/goenv-latest" --print "$go_version") ynh_print_info "Installation of Go-$final_go_version" goenv install --quiet --skip-existing "$final_go_version" 2>&1