diff --git a/helpers/helpers.v2.1.d/apps b/helpers/helpers.v2.1.d/apps new file mode 100644 index 000000000..81a5717eb --- /dev/null +++ b/helpers/helpers.v2.1.d/apps @@ -0,0 +1,215 @@ +#!/bin/bash + +# Install others YunoHost apps +# +# usage: ynh_install_apps --apps="appfoo?domain=domain.foo&path=/foo appbar?domain=domain.bar&path=/bar&admin=USER&language=fr&is_public=1&pass?word=pass&port=666" +# | arg: -a, --apps= - apps to install +# +# Requires YunoHost version *.*.* or higher. +ynh_install_apps() { + # Declare an array to define the options of this helper. + local legacy_args=a + local -A args_array=([a]=apps=) + local apps + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Split the list of apps in an array + local apps_list=($(echo $apps | tr " " "\n")) + local apps_dependencies="" + + # For each app + for one_app_and_its_args in "${apps_list[@]}" + do + # Retrieve the name of the app (part before ?) + local one_app=$(cut -d "?" -f1 <<< "$one_app_and_its_args") + [ -z "$one_app" ] && ynh_die --message="You didn't provided a YunoHost app to install" + + yunohost tools update apps + + # Installing or upgrading the app depending if it's installed or not + if ! yunohost app list --output-as json --quiet | jq -e --arg id $one_app '.apps[] | select(.id == $id)' >/dev/null + then + # Retrieve the arguments of the app (part after ?) + local one_argument="" + if [[ "$one_app_and_its_args" == *"?"* ]]; then + one_argument=$(cut -d "?" -f2- <<< "$one_app_and_its_args") + one_argument="--args $one_argument" + fi + + # Install the app with its arguments + yunohost app install $one_app $one_argument + else + # Upgrade the app + yunohost app upgrade $one_app + fi + + if [ ! -z "$apps_dependencies" ] + then + apps_dependencies="$apps_dependencies, $one_app" + else + apps_dependencies="$one_app" + fi + done + + ynh_app_setting_set --app=$app --key=apps_dependencies --value="$apps_dependencies" +} + +# Remove other YunoHost apps +# +# Other YunoHost apps will be removed only if no other apps need them. +# +# usage: ynh_remove_apps +# +# Requires YunoHost version *.*.* or higher. +ynh_remove_apps() { + # Retrieve the apps dependencies of the app + local apps_dependencies=$(ynh_app_setting_get --app=$app --key=apps_dependencies) + ynh_app_setting_delete --app=$app --key=apps_dependencies + + if [ ! -z "$apps_dependencies" ] + then + # Split the list of apps dependencies in an array + local apps_dependencies_list=($(echo $apps_dependencies | tr ", " "\n")) + + # For each apps dependencies + for one_app in "${apps_dependencies_list[@]}" + do + # Retrieve the list of installed apps + local installed_apps_list=$(yunohost app list --output-as json --quiet | jq -r .apps[].id) + local required_by="" + local installed_app_required_by="" + + # For each other installed app + for one_installed_app in $installed_apps_list + do + # Retrieve the other apps dependencies + one_installed_apps_dependencies=$(ynh_app_setting_get --app=$one_installed_app --key=apps_dependencies) + if [ ! -z "$one_installed_apps_dependencies" ] + then + one_installed_apps_dependencies_list=($(echo $one_installed_apps_dependencies | tr ", " "\n")) + + # For each dependency of the other apps + for one_installed_app_dependency in "${one_installed_apps_dependencies_list[@]}" + do + if [[ $one_installed_app_dependency == $one_app ]]; then + required_by="$required_by $one_installed_app" + fi + done + fi + done + + # If $one_app is no more required + if [[ -z "$required_by" ]] + then + # Remove $one_app + ynh_print_info --message="Removing of $one_app" + yunohost app remove $one_app --purge + else + ynh_print_info --message="$one_app was not removed because it's still required by${required_by}" + fi + done + fi +} + +# Spawn a Bash shell with the app environment loaded +# +# usage: ynh_spawn_app_shell --app="app" +# | arg: -a, --app= - the app ID +# +# examples: +# ynh_spawn_app_shell --app="APP" <<< 'echo "$USER"' +# ynh_spawn_app_shell --app="APP" < /tmp/some_script.bash +# +# Requires YunoHost version 11.0.* or higher, and that the app relies on packaging v2 or higher. +# The spawned shell will have environment variables loaded and environment files sourced +# from the app's service configuration file (defaults to $app.service, overridable by the packager with `service` setting). +# If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings. +ynh_spawn_app_shell() { + # Declare an array to define the options of this helper. + local legacy_args=a + local -A args_array=([a]=app=) + local app + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Force Bash to be used to run this helper + if [[ ! $0 =~ \/?bash$ ]] + then + ynh_print_err --message="Please use Bash as shell" + exit 1 + fi + + # Make sure the app is installed + local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id)) + if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]] + then + ynh_print_err --message="$app is not in the apps list" + exit 1 + fi + + # Make sure the app has its own user + if ! id -u "$app" &>/dev/null; then + ynh_print_err --message="There is no \"$app\" system user" + exit 1 + fi + + # Make sure the app has an install_dir setting + local install_dir=$(ynh_app_setting_get --app=$app --key=install_dir) + if [ -z "$install_dir" ] + then + ynh_print_err --message="$app has no install_dir setting (does it use packaging format >=2?)" + exit 1 + fi + + # Load the app's service name, or default to $app + local service=$(ynh_app_setting_get --app=$app --key=service) + [ -z "$service" ] && service=$app; + + # Export HOME variable + export HOME=$install_dir; + + # Load the Environment variables from the app's service + local env_var=$(systemctl show $service.service -p "Environment" --value) + [ -n "$env_var" ] && export $env_var; + + # Force `php` to its intended version + # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` + local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + local phpflags=$(ynh_app_setting_get --app=$app --key=phpflags) + if [ -n "$phpversion" ] + then + eval "php() { php${phpversion} ${phpflags} \"\$@\"; }" + export -f php + fi + + # Source the EnvironmentFiles from the app's service + local env_files=($(systemctl show $service.service -p "EnvironmentFiles" --value)) + if [ ${#env_files[*]} -gt 0 ] + then + # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. + set -a + for file in ${env_files[*]} + do + [[ $file = /* ]] && source $file + done + set +a + fi + + # Activate the Python environment, if it exists + if [ -f $install_dir/venv/bin/activate ] + then + # set -/+a enables and disables new variables being automatically exported. Needed when using `source`. + set -a + source $install_dir/venv/bin/activate + set +a + fi + + # cd into the WorkingDirectory set in the service, or default to the install_dir + local env_dir=$(systemctl show $service.service -p "WorkingDirectory" --value) + [ -z $env_dir ] && env_dir=$install_dir; + cd $env_dir + + # Spawn the app shell + su -s /bin/bash $app +} diff --git a/helpers/helpers.v2.1.d/apt b/helpers/helpers.v2.1.d/apt new file mode 100644 index 000000000..d5a1f4335 --- /dev/null +++ b/helpers/helpers.v2.1.d/apt @@ -0,0 +1,636 @@ +#!/bin/bash + +# Check if apt is free to use, or wait, until timeout. +# +# [internal] +# +# usage: ynh_wait_dpkg_free +# | exit: Return 1 if dpkg is broken +# +# Requires YunoHost version 3.3.1 or higher. +ynh_wait_dpkg_free() { + local try + set +o xtrace # set +x + # With seq 1 17, timeout will be almost 30 minutes + for try in $(seq 1 17); do + # Check if /var/lib/dpkg/lock is used by another process + if lsof /var/lib/dpkg/lock >/dev/null; then + echo "apt is already in use..." + # Sleep an exponential time at each round + sleep $((try * try)) + else + # Check if dpkg hasn't been interrupted and is fully available. + # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + local dpkg_dir="/var/lib/dpkg/updates/" + + # For each file in $dpkg_dir + while read dpkg_file <&9; do + # Check if the name of this file contains only numbers. + if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then + # If so, that a remaining of dpkg. + ynh_print_err "dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + set -o xtrace # set -x + return 1 + fi + done 9<<<"$(ls -1 $dpkg_dir)" + set -o xtrace # set -x + return 0 + fi + done + echo "apt still used, but timeout reached !" + set -o xtrace # set -x +} + +# Check either a package is installed or not +# +# example: ynh_package_is_installed --package=yunohost && echo "installed" +# +# usage: ynh_package_is_installed --package=name +# | arg: -p, --package= - the package name to check +# | ret: 0 if the package is installed, 1 else. +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_is_installed() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=package=) + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + dpkg-query --show --showformat='${Status}' "$package" 2>/dev/null \ + | grep --count "ok installed" &>/dev/null +} + +# Get the version of an installed package +# +# example: version=$(ynh_package_version --package=yunohost) +# +# [internal] +# +# usage: ynh_package_version --package=name +# | arg: -p, --package= - the package name to get version +# | ret: the version or an empty string +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_version() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=package=) + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_package_is_installed "$package"; then + dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null + else + echo '' + fi +} + +# APT wrapper for non-interactive operation +# +# [internal] +# +# usage: ynh_apt update +# +# Requires YunoHost version 2.4.0.3 or higher. +ynh_apt() { + ynh_wait_dpkg_free + LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0 $@ +} + +# Update package index files +# +# [internal] +# +# usage: ynh_package_update +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_update() { + ynh_apt update +} + +# Install package(s) +# +# [internal] +# +# usage: ynh_package_install name [name [...]] +# | arg: name - the package name to install +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_install() { + ynh_apt --no-remove --option Dpkg::Options::=--force-confdef \ + --option Dpkg::Options::=--force-confold install $@ +} + +# Remove package(s) +# +# [internal] +# +# usage: ynh_package_remove name [name [...]] +# | arg: name - the package name to remove +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_remove() { + ynh_apt remove $@ +} + +# Remove package(s) and their uneeded dependencies +# +# [internal] +# +# usage: ynh_package_autoremove name [name [...]] +# | arg: name - the package name to remove +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_autoremove() { + ynh_apt autoremove $@ +} + +# Purge package(s) and their uneeded dependencies +# +# [internal] +# +# usage: ynh_package_autopurge name [name [...]] +# | arg: name - the package name to autoremove and purge +# +# Requires YunoHost version 2.7.2 or higher. +ynh_package_autopurge() { + ynh_apt autoremove --purge $@ +} + +# Build and install a package from an equivs control file +# +# [internal] +# +# example: generate an empty control file with `equivs-control`, adjust its +# content and use helper to build and install the package: +# ynh_package_install_from_equivs /path/to/controlfile +# +# usage: ynh_package_install_from_equivs controlfile +# | arg: controlfile - path of the equivs control file +# +# Requires YunoHost version 2.2.4 or higher. +ynh_package_install_from_equivs() { + local controlfile=$1 + + # retrieve package information + local pkgname=$(grep '^Package: ' $controlfile | cut --delimiter=' ' --fields=2) # Retrieve the name of the debian package + local pkgversion=$(grep '^Version: ' $controlfile | cut --delimiter=' ' --fields=2) # And its version number + [[ -z "$pkgname" || -z "$pkgversion" ]] \ + && ynh_die --message="Invalid control file" # Check if this 2 variables aren't empty. + + # Update packages cache + ynh_package_update + + # Build and install the package + local TMPDIR=$(mktemp --directory) + + # Make sure to delete the legacy compat file + # It's now handle somewhat magically through the control file + rm -f /usr/share/equivs/template/debian/compat + + # Note that the cd executes into a sub shell + # Create a fake deb package with equivs-build and the given control file + # Install the fake package without its dependencies with dpkg + # Install missing dependencies with ynh_package_install + ynh_wait_dpkg_free + cp "$controlfile" "${TMPDIR}/control" + ( + cd "$TMPDIR" + LC_ALL=C equivs-build ./control 2>&1 + LC_ALL=C dpkg --force-depends --install "./${pkgname}_${pkgversion}_all.deb" 2>&1 | tee ./dpkg_log + ) + + ynh_package_install --fix-broken \ + || { # If the installation failed + # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process) + # Parse the list of problematic dependencies from dpkg's log ... + # (relevant lines look like: "foo-ynh-deps depends on bar; however:") + local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')" + # Fake an install of those dependencies to see the errors + # The sed command here is, Print only from 'Reading state info' to the end. + [[ -n "$problematic_dependencies" ]] && ynh_package_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2 + ynh_die --message="Unable to install dependencies" + } + [[ -n "$TMPDIR" ]] && rm --recursive --force $TMPDIR # Remove the temp dir. + + # check if the package is actually installed + ynh_package_is_installed "$pkgname" +} + +YNH_INSTALL_APP_DEPENDENCIES_REPLACE="true" + +# Define and install dependencies with a equivs control file +# +# [packagingv1] +# +# This helper can/should only be called once per app +# +# example : ynh_install_app_dependencies dep1 dep2 "dep3|dep4|dep5" +# +# usage: ynh_install_app_dependencies dep [dep [...]] +# | arg: dep - the package name to install in dependence. +# | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc). +# +# Requires YunoHost version 2.6.4 or higher. +ynh_install_app_dependencies() { + local dependencies=$@ + # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) + dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" + local dependencies=${dependencies//|/ | } + + local version=$(ynh_read_manifest --manifest_key="version") + if [ -z "${version}" ] || [ "$version" == "null" ]; then + version="1.0" + fi + local dep_app=${app//_/-} # Replace all '_' by '-' + + # Handle specific versions + if [[ "$dependencies" =~ [\<=\>] ]]; then + # Replace version specifications by relationships syntax + # https://www.debian.org/doc/debian-policy/ch-relationships.html + # Sed clarification + # [^(\<=\>] ignore if it begins by ( or < = >. To not apply twice. + # [\<=\>] matches < = or > + # \+ matches one or more occurence of the previous characters, for >= or >>. + # [^,]\+ matches all characters except ',' + # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)' + dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')" + fi + + # Check for specific php dependencies which requires sury + # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni" + # The (?<=php) syntax corresponds to lookbehind ;) + local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u) + + if [[ -n "$specific_php_version" ]] + then + # Cover a small edge case where a packager could have specified "php7.4-pwet php5-gni" which is confusing + [[ $(echo $specific_php_version | wc -l) -eq 1 ]] \ + || ynh_die --message="Inconsistent php versions in dependencies ... found : $specific_php_version" + + dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common" + + local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + + # If the PHP version changed, remove the old fpm conf + if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$specific_php_version" ]; then + local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" + + if [[ -f "$old_php_finalphpconf" ]] + then + ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf" + ynh_remove_fpm_config + fi + fi + # Store phpversion into the config of this app + ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version + + # Set the default php version back as the default version for php-cli. + if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION + then + update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION + fi + elif grep --quiet 'php' <<< "$dependencies"; then + ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION + fi + + local psql_installed="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)" + + # The first time we run ynh_install_app_dependencies, we will replace the + # entire control file (This is in particular meant to cover the case of + # upgrade script where ynh_install_app_dependencies is called with this + # expected effect) Otherwise, any subsequent call will add dependencies + # to those already present in the equivs control file. + if [[ $YNH_INSTALL_APP_DEPENDENCIES_REPLACE == "true" ]] + then + YNH_INSTALL_APP_DEPENDENCIES_REPLACE="false" + else + local current_dependencies="" + if ynh_package_is_installed --package="${dep_app}-ynh-deps" + then + current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) " + current_dependencies=${current_dependencies// | /|} + fi + dependencies="$current_dependencies, $dependencies" + fi + + cat >/tmp/${dep_app}-ynh-deps.control </dev/null + then + ynh_package_autopurge ${dep_app}-ynh-deps + fi +} + +# Install packages from an extra repository properly. +# +# [packagingv1] +# +# usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" [--key=key_url] [--name=name] +# | arg: -r, --repo= - Complete url of the extra repository. +# | arg: -p, --package= - The packages to install from this extra repository +# | arg: -k, --key= - url to get the public key. +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_install_extra_app_dependencies() { + # Declare an array to define the options of this helper. + local legacy_args=rpkn + local -A args_array=([r]=repo= [p]=package= [k]=key= [n]=name=) + local repo + local package + local key + local name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + key=${key:-} + + # Set a key only if asked + if [ -n "$key" ]; then + key="--key=$key" + fi + # Add an extra repository for those packages + ynh_install_extra_repo --repo="$repo" $key --priority=995 --name=$name + + # Install requested dependencies from this extra repository. + ynh_install_app_dependencies "$package" + + # Force to upgrade to the last version... + # Without doing apt install, an already installed dep is not upgraded + local apps_auto_installed="$(apt-mark showauto $package)" + ynh_package_install "$package" + [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed + + # Remove this extra repository after packages are installed + ynh_remove_extra_repo --name=$name +} + +# Add an extra repository correctly, pin it and get the key. +# +# [internal] +# +# usage: ynh_install_extra_repo --repo="repo" [--key=key_url] [--priority=priority_value] [--name=name] [--append] +# | arg: -r, --repo= - Complete url of the extra repository. +# | arg: -k, --key= - url to get the public key. +# | arg: -p, --priority= - Priority for the pin +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_install_extra_repo() { + # Declare an array to define the options of this helper. + local legacy_args=rkpna + local -A args_array=([r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append) + local repo + local key + local priority + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + append=${append:-0} + key=${key:-} + priority=${priority:-} + + if [ $append -eq 1 ]; then + append="--append" + wget_append="tee --append" + else + append="" + wget_append="tee" + fi + + # Split the repository into uri, suite and components. + # Remove "deb " at the beginning of the repo. + repo="${repo#deb }" + + # Get the uri + local uri="$(echo "$repo" | awk '{ print $1 }')" + + # Get the suite + local suite="$(echo "$repo" | awk '{ print $2 }')" + + # Get the components + local component="${repo##$uri $suite }" + + # Add the repository into sources.list.d + ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append + + # Pin the new repo with the default priority, so it won't be used for upgrades. + # Build $pin from the uri without http and any sub path + local pin="${uri#*://}" + pin="${pin%%/*}" + # Set a priority only if asked + if [ -n "$priority" ]; then + priority="--priority=$priority" + fi + ynh_pin_repo --package="*" --pin="origin \"$pin\"" $priority --name="$name" $append + + # Get the public key for the repo + if [ -n "$key" ]; then + mkdir --parents "/etc/apt/trusted.gpg.d" + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg >/dev/null + fi + + # Update the list of package with the new repo + ynh_package_update +} + +# Remove an extra repository and the assiociated configuration. +# +# [internal] +# +# usage: ynh_remove_extra_repo [--name=name] +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_remove_extra_repo() { + # Declare an array to define the options of this helper. + local legacy_args=n + local -A args_array=([n]=name=) + local name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + + ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list" + # Sury pinning is managed by the regenconf in the core... + [[ "$name" == "extra_php_version" ]] || ynh_secure_remove "/etc/apt/preferences.d/$name" + if [ -e /etc/apt/trusted.gpg.d/$name.gpg ]; then + ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg" + fi + + # (Do we even create a .asc file anywhere ...?) + if [ -e /etc/apt/trusted.gpg.d/$name.asc ]; then + ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.asc" + fi + + # Update the list of package to exclude the old repo + ynh_package_update +} + +# Add a repository. +# +# [internal] +# +# usage: ynh_add_repo --uri=uri --suite=suite --component=component [--name=name] [--append] +# | arg: -u, --uri= - Uri of the repository. +# | arg: -s, --suite= - Suite of the repository. +# | arg: -c, --component= - Component of the repository. +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# Example for a repo like deb http://forge.yunohost.org/debian/ stretch stable +# uri suite component +# ynh_add_repo --uri=http://forge.yunohost.org/debian/ --suite=stretch --component=stable +# +# Requires YunoHost version 3.8.1 or higher. +ynh_add_repo() { + # Declare an array to define the options of this helper. + local legacy_args=uscna + local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append) + local uri + local suite + local component + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + append=${append:-0} + + if [ $append -eq 1 ]; then + append="tee --append" + else + append="tee" + fi + + mkdir --parents "/etc/apt/sources.list.d" + # Add the new repo in sources.list.d + echo "deb $uri $suite $component" \ + | $append "/etc/apt/sources.list.d/$name.list" +} + +# Pin a repository. +# +# [internal] +# +# usage: ynh_pin_repo --package=packages --pin=pin_filter [--priority=priority_value] [--name=name] [--append] +# | arg: -p, --package= - Packages concerned by the pin. Or all, *. +# | arg: -i, --pin= - Filter for the pin. +# | arg: -p, --priority= - Priority for the pin +# | arg: -n, --name= - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# See https://manpages.debian.org/stretch/apt/apt_preferences.5.en.html#How_APT_Interprets_Priorities for information about pinning. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_pin_repo() { + # Declare an array to define the options of this helper. + local legacy_args=pirna + local -A args_array=([p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append) + local package + local pin + local priority + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package="${package:-*}" + priority=${priority:-50} + name="${name:-$app}" + append=${append:-0} + + if [ $append -eq 1 ]; then + append="tee --append" + else + append="tee" + fi + + # Sury pinning is managed by the regenconf in the core... + [[ "$name" != "extra_php_version" ]] || return 0 + + mkdir --parents "/etc/apt/preferences.d" + echo "Package: $package +Pin: $pin +Pin-Priority: $priority +" \ + | $append "/etc/apt/preferences.d/$name" +} diff --git a/helpers/helpers.v2.1.d/backup b/helpers/helpers.v2.1.d/backup new file mode 100644 index 000000000..a596ac9e0 --- /dev/null +++ b/helpers/helpers.v2.1.d/backup @@ -0,0 +1,507 @@ +#!/bin/bash + +CAN_BIND=${CAN_BIND:-1} + +# Add a file or a directory to the list of paths to backup +# +# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory] +# | arg: -s, --src_path= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. +# | arg: -d, --dest_path= - destination file or directory inside the backup dir +# | arg: -b, --is_big - Indicate data are big (mail, video, image ...) +# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it. +# +# This helper can be used both in a system backup hook, and in an app backup script +# +# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it +# creates the parent destination directory +# +# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`. +# +# Example in the context of a wordpress app : +# ``` +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" +# # => This line will be added into CSV file +# # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf" +# +# #Deprecated usages (maintained for retro-compatibility) +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" +# +# ``` +# +# How to use `--is_big`: +# +# `--is_big` is used to specify that this part of the backup can be quite huge. +# So, you don't want that your package does backup that part during ynh_backup_before_upgrade. +# In the same way, an user may doesn't want to backup this big part of the app for +# each of his backup. And so handle that part differently. +# +# As this part of your backup may not be done, your restore script has to handle it. +# In your restore script, use `--not_mandatory` with `ynh_restore_file` +# As well in your remove script, you should not remove those data ! Or an user may end up with +# a failed upgrade restoring an app without data anymore ! +# +# To have the benefit of `--is_big` while doing a backup, you can whether set the environement +# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect +# only that backup command. +# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect +# all backups for this app until the setting is removed. +# +# Requires YunoHost version 2.4.0 or higher. +# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory` +ynh_backup() { + # TODO find a way to avoid injection by file strange naming ! + + # Declare an array to define the options of this helper. + local legacy_args=sdbm + local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory) + local src_path + local dest_path + local is_big + local not_mandatory + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + dest_path="${dest_path:-}" + is_big="${is_big:-0}" + not_mandatory="${not_mandatory:-0}" + + BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} + test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data) + + # If backing up core only (used by ynh_backup_before_upgrade), + # don't backup big data items + if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then + if [ $BACKUP_CORE_ONLY -eq 1 ]; then + ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set." + else + ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set." + fi + return 0 + fi + + # ============================================================================== + # Format correctly source and destination paths + # ============================================================================== + # Be sure the source path is not empty + if [ ! -e "$src_path" ]; then + ynh_print_warn --message="Source path '${src_path}' does not exist" + if [ "$not_mandatory" == "0" ]; then + # This is a temporary fix for fail2ban config files missing after the migration to stretch. + if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then + touch "${src_path}" + ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!" + else + return 1 + fi + else + return 0 + fi + fi + + # Transform the source path as an absolute path + # If it's a dir remove the ending / + src_path=$(realpath "$src_path") + + # If there is no destination path, initialize it with the source path + # relative to "/". + # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost + if [[ -z "$dest_path" ]]; then + dest_path="${src_path#/}" + + else + if [[ "${dest_path:0:1}" == "/" ]]; then + + # If the destination path is an absolute path, transform it as a path + # relative to the current working directory ($YNH_CWD) + # + # If it's an app backup script that run this helper, YNH_CWD is equal to + # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/ + # + # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR + dest_path="${dest_path#$YNH_CWD/}" + + # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD + if [[ "${dest_path:0:1}" == "/" ]]; then + dest_path="${dest_path#/}" + fi + fi + + # Complete dest_path if ended by a / + if [[ "${dest_path: -1}" == "/" ]]; then + dest_path="${dest_path}/$(basename $src_path)" + fi + fi + + # Check if dest_path already exists in tmp archive + if [[ -e "${dest_path}" ]]; then + ynh_print_err --message="Destination path '${dest_path}' already exist" + return 1 + fi + + # Add the relative current working directory to the destination path + local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}" + rel_dir="${rel_dir%/}/" + dest_path="${rel_dir}${dest_path}" + dest_path="${dest_path#/}" + # ============================================================================== + + # ============================================================================== + # Write file to backup into backup_list + # ============================================================================== + local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g') + local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g') + echo "\"${src}\",\"${dest}\"" >>"${YNH_BACKUP_CSV}" + + # ============================================================================== + + # Create the parent dir of the destination path + # It's for retro compatibility, some script consider ynh_backup creates this dir + mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}") +} + +# Restore all files that were previously backuped in a core backup script or app backup script +# +# usage: ynh_restore +# +# Requires YunoHost version 2.6.4 or higher. +ynh_restore() { + # Deduce the relative path of $YNH_CWD + local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}" + REL_DIR="${REL_DIR%/}/" + + # For each destination path begining by $REL_DIR + cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \ + | while read line; do + local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)") + local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)") + ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH" + done +} + +# Return the path in the archive where has been stocked the origin path +# +# [internal] +# +# usage: _get_archive_path ORIGIN_PATH +_get_archive_path() { + # For security reasons we use csv python library to read the CSV + python3 -c " +import sys +import csv +with open(sys.argv[1], 'r') as backup_file: + backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest']) + for row in backup_csv: + if row['source']==sys.argv[2].strip('\"'): + print(row['dest']) + sys.exit(0) + raise Exception('Original path for %s not found' % sys.argv[2]) + " "${YNH_BACKUP_CSV}" "$1" + return $? +} + +# Restore a file or a directory +# +# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory] +# | arg: -o, --origin_path= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive +# | arg: -d, --dest_path= - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv` +# | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it. +# +# Use the registered path in backup_list by ynh_backup to restore the file at the right place. +# +# examples: +# ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf" +# # You can also use relative paths: +# ynh_restore_file -o "conf/nginx.conf" +# +# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in +# `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed. +# +# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into +# `/etc/nginx/conf.d/$domain.d/$app.conf` +# if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into +# `/etc/nginx/conf.d/$domain.d/$app.conf` +# +# Requires YunoHost version 2.6.4 or higher. +# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory +ynh_restore_file() { + # Declare an array to define the options of this helper. + local legacy_args=odm + local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory) + local origin_path + local dest_path + local not_mandatory + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + origin_path="/${origin_path#/}" + # Default value for dest_path = /$origin_path + dest_path="${dest_path:-$origin_path}" + not_mandatory="${not_mandatory:-0}" + + local archive_path="$YNH_CWD${origin_path}" + # If archive_path doesn't exist, search for a corresponding path in CSV + if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then + if [ "$not_mandatory" == "0" ]; then + archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")" + else + return 0 + fi + fi + + # Move the old directory if it already exists + if [[ -e "${dest_path}" ]]; then + # Check if the file/dir size is less than 500 Mo + if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then + local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')" + mkdir --parents "$(dirname "$backup_file")" + mv "${dest_path}" "$backup_file" # Move the current file or directory + else + ynh_secure_remove --file=${dest_path} + fi + fi + + # Restore origin_path into dest_path + mkdir --parents $(dirname "$dest_path") + + # Do a copy if it's just a mounting point + if mountpoint --quiet $YNH_BACKUP_DIR; then + if [[ -d "${archive_path}" ]]; then + archive_path="${archive_path}/." + mkdir --parents "$dest_path" + fi + cp --archive "$archive_path" "${dest_path}" + # Do a move if YNH_BACKUP_DIR is already a copy + else + mv "$archive_path" "${dest_path}" + fi + + # Boring hack for nginx conf file mapped to php7.3 + # Note that there's no need to patch the fpm config because most php apps + # will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch + if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}" + then + sed -i 's/php7.3/php7.4/g' "${dest_path}" + fi +} + +# Calculate and store a file checksum into the app settings +# +# usage: ynh_store_file_checksum --file=file +# | arg: -f, --file= - The file on which the checksum will performed, then stored. +# +# $app should be defined when calling this helper +# +# Requires YunoHost version 2.6.4 or higher. +ynh_store_file_checksum() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file= [u]=update_only) + local file + local update_only + update_only="${update_only:-0}" + + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + + # If update only, we don't save the new checksum if no old checksum exist + if [ $update_only -eq 1 ]; then + local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + if [ -z "${checksum_value}" ]; then + unset backup_file_checksum + return 0 + fi + fi + + ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) + + if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ... + local file_path_base64=$(echo "$file" | base64 -w0) + mkdir -p /var/cache/yunohost/appconfbackup/ + cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64} + fi + + # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup + if [ -n "${backup_file_checksum-}" ]; then + # Print the diff between the previous file and the new one. + # diff return 1 if the files are different, so the || true + diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true + fi + # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. + unset backup_file_checksum +} + +# Verify the checksum and backup the file if it's different +# +# usage: ynh_backup_if_checksum_is_different --file=file +# | arg: -f, --file= - The file on which the checksum test will be perfomed. +# | ret: the name of a backup file, or nothing +# +# This helper is primarily meant to allow to easily backup personalised/manually +# modified config files. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_backup_if_checksum_is_different() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum + backup_file_checksum="" + if [ -n "$checksum_value" ]; then # Proceed only if a value was stored into the app settings + if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different + + backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')" + mkdir --parents "$(dirname "$backup_file_checksum")" + cp --archive "$file" "$backup_file_checksum" # Backup the current file + ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + echo "$backup_file_checksum" # Return the name of the backup file + if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + local file_path_base64=$(echo "$file" | base64 -w0) + if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64} + then + ynh_print_warn "Diff with the original file:" + diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true + fi + fi + fi + fi +} + +# Delete a file checksum from the app settings +# +# usage: ynh_delete_file_checksum --file=file +# | arg: -f, --file= - The file for which the checksum will be deleted +# +# $app should be defined when calling this helper +# +# Requires YunoHost version 3.3.1 or higher. +ynh_delete_file_checksum() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + ynh_app_setting_delete --app=$app --key=$checksum_setting_name +} + +# Checks a backup archive exists +# +# [internal] +# +ynh_backup_archive_exists() { + yunohost backup list --output-as json --quiet \ + | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null +} + +# Make a backup in case of failed upgrade +# +# [packagingv1] +# +# usage: ynh_backup_before_upgrade +# +# Usage in a package script: +# ``` +# ynh_backup_before_upgrade +# ynh_clean_setup () { +# ynh_restore_upgradebackup +# } +# ynh_abort_if_errors +# ``` +# +# Requires YunoHost version 2.7.2 or higher. +ynh_backup_before_upgrade() { + if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]; then + ynh_print_warn --message="This app doesn't have any backup script." + return + fi + backup_number=1 + local old_backup_number=2 + local app_bck=${app//_/-} # Replace all '_' by '-' + NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} + + if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then + # Check if a backup already exists with the prefix 1 + if ynh_backup_archive_exists "$app_bck-pre-upgrade1"; then + # Prefix becomes 2 to preserve the previous backup + backup_number=2 + old_backup_number=1 + fi + + # Create backup + BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug + if [ "$?" -eq 0 ]; then + # If the backup succeeded, remove the previous backup + if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then + # Remove the previous backup only if it exists + yunohost backup delete $app_bck-pre-upgrade$old_backup_number >/dev/null + fi + else + ynh_die --message="Backup failed, the upgrade process was aborted." + fi + else + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" + fi +} + +# Restore a previous backup if the upgrade process failed +# +# [packagingv1] +# +# usage: ynh_restore_upgradebackup +# +# Usage in a package script: +# ``` +# ynh_backup_before_upgrade +# ynh_clean_setup () { +# ynh_restore_upgradebackup +# } +# ynh_abort_if_errors +# ``` +# +# Requires YunoHost version 2.7.2 or higher. +ynh_restore_upgradebackup() { + ynh_print_err --message="Upgrade failed." + local app_bck=${app//_/-} # Replace all '_' by '-' + + NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} + + if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then + # Check if an existing backup can be found before removing and restoring the application. + if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"; then + # Remove the application then restore it + yunohost app remove $app + # Restore the backup + yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug + if [[ -d /etc/yunohost/apps/$app ]] + then + ynh_die --message="The app was restored to the way it was before the failed upgrade." + else + ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|" + fi + fi + else + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" + fi +} diff --git a/helpers/helpers.v2.1.d/config b/helpers/helpers.v2.1.d/config new file mode 100644 index 000000000..d2cc2760e --- /dev/null +++ b/helpers/helpers.v2.1.d/config @@ -0,0 +1,354 @@ +#!/bin/bash + +_ynh_app_config_get_one() { + local short_setting="$1" + local type="$2" + local bind="$3" + local getter="get__${short_setting}" + # Get value from getter if exists + if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + old[$short_setting]="$($getter)" + formats[${short_setting}]="yaml" + + elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + old[$short_setting]="$("get__${bind%%(*}" $short_setting $type $bind)" + formats[${short_setting}]="yaml" + + elif [[ "$bind" == "null" ]]; then + old[$short_setting]="YNH_NULL" + + # Get value from app settings or from another file + elif [[ "$type" == "file" ]]; then + if [[ "$bind" == "settings" ]]; then + ynh_die --message="File '${short_setting}' can't be stored in settings" + fi + old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" + file_hash[$short_setting]="true" + + # Get multiline text from settings or from a full file + elif [[ "$type" == "text" ]]; then + if [[ "$bind" == "settings" ]]; then + old[$short_setting]="$(ynh_app_setting_get $app $short_setting)" + elif [[ "$bind" == *":"* ]]; then + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + else + old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" + fi + + # Get value from a kind of key/value file + else + local bind_after="" + if [[ "$bind" == "settings" ]]; then + bind=":/etc/yunohost/apps/$app/settings.yml" + fi + local bind_key_="$(echo "$bind" | cut -d: -f1)" + bind_key_=${bind_key_:-$short_setting} + if [[ "$bind_key_" == *">"* ]]; then + bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" + bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" + fi + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key_}" --after="${bind_after}")" + + fi +} +_ynh_app_config_apply_one() { + local short_setting="$1" + local setter="set__${short_setting}" + local bind="${binds[$short_setting]}" + local type="${types[$short_setting]}" + if [ "${changed[$short_setting]}" == "true" ]; then + # Apply setter if exists + if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then + $setter + + elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + "set__${bind%%(*}" $short_setting $type $bind + + elif [[ "$bind" == "null" ]]; then + return + + # Save in a file + elif [[ "$type" == "file" ]]; then + if [[ "$bind" == "settings" ]]; then + ynh_die --message="File '${short_setting}' can't be stored in settings" + fi + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + if [[ "${!short_setting}" == "" ]]; then + ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_secure_remove --file="$bind_file" + ynh_delete_file_checksum --file="$bind_file" + ynh_print_info --message="File '$bind_file' removed" + else + ynh_backup_if_checksum_is_different --file="$bind_file" + if [[ "${!short_setting}" != "$bind_file" ]]; then + cp "${!short_setting}" "$bind_file" + fi + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info --message="File '$bind_file' overwritten with ${!short_setting}" + fi + + # Save value in app settings + elif [[ "$bind" == "settings" ]]; then + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_print_info --message="Configuration key '$short_setting' edited in app settings" + + # Save multiline text in a file + elif [[ "$type" == "text" ]]; then + if [[ "$bind" == *":"* ]]; then + ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" + fi + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + ynh_backup_if_checksum_is_different --file="$bind_file" + echo "${!short_setting}" >"$bind_file" + ynh_store_file_checksum --file="$bind_file" --update_only + ynh_print_info --message="File '$bind_file' overwritten with the content provided in question '${short_setting}'" + + # Set value into a kind of key/value file + else + local bind_after="" + local bind_key_="$(echo "$bind" | cut -d: -f1)" + if [[ "$bind_key_" == *">"* ]]; then + bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" + bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" + fi + bind_key_=${bind_key_:-$short_setting} + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + + ynh_backup_if_checksum_is_different --file="$bind_file" + ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}" + ynh_store_file_checksum --file="$bind_file" --update_only + + # We stored the info in settings in order to be able to upgrade the app + ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}" + ynh_print_info --message="Configuration key '$bind_key_' edited into $bind_file" + + fi + fi +} +_ynh_app_config_get() { + # From settings + local lines + lines=$( + python3 <" in bind_section: + bind_section = bind_section + bind_panel_file + else: + bind_section = regex + bind_section + bind_panel_file + + for name, param in section.items(): + if not isinstance(param, dict): + continue + + bind = param.get('bind') + + if not bind: + if bind_section: + bind = bind_section + else: + bind = 'settings' + elif bind[-1] == ":" and bind_section and ":" in bind_section: + regex, bind_file = bind_section.split(":") + if ">" in bind: + bind = bind + bind_file + else: + bind = regex + bind + bind_file + if bind == "settings" and param.get('type', 'string') == 'file': + bind = 'null' + + print('|'.join([ + name, + param.get('type', 'string'), + bind + ])) +EOL + ) + for line in $lines; do + # Split line into short_setting, type and bind + IFS='|' read short_setting type bind <<<"$line" + binds[${short_setting}]="$bind" + types[${short_setting}]="$type" + file_hash[${short_setting}]="" + formats[${short_setting}]="" + ynh_app_config_get_one $short_setting $type $bind + done + +} + +_ynh_app_config_apply() { + for short_setting in "${!old[@]}"; do + ynh_app_config_apply_one $short_setting + done +} + +_ynh_app_config_show() { + for short_setting in "${!old[@]}"; do + if [[ "${old[$short_setting]}" != YNH_NULL ]]; then + if [[ "${formats[$short_setting]}" == "yaml" ]]; then + ynh_return "${short_setting}:" + ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/ /g')" + else + ynh_return "${short_setting}: '$(echo "${old[$short_setting]}" | sed "s/'/''/g" | sed ':a;N;$!ba;s/\n/\n\n/g')'" + fi + fi + done +} + +_ynh_app_config_validate() { + # Change detection + ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1 + local nothing_changed=true + local changes_validated=true + for short_setting in "${!old[@]}"; do + changed[$short_setting]=false + if [ -z ${!short_setting+x} ]; then + # Assign the var with the old value in order to allows multiple + # args validation + declare -g "$short_setting"="${old[$short_setting]}" + continue + fi + if [ ! -z "${file_hash[${short_setting}]}" ]; then + file_hash[old__$short_setting]="" + file_hash[new__$short_setting]="" + if [ -f "${old[$short_setting]}" ]; then + file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1) + if [ -z "${!short_setting}" ]; then + changed[$short_setting]=true + nothing_changed=false + fi + fi + if [ -f "${!short_setting}" ]; then + file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1) + if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]; then + changed[$short_setting]=true + nothing_changed=false + fi + fi + else + if [[ "${!short_setting}" != "${old[$short_setting]}" ]]; then + changed[$short_setting]=true + nothing_changed=false + fi + fi + done + if [[ "$nothing_changed" == "true" ]]; then + ynh_print_info --message="Nothing has changed" + exit 0 + fi + + # Run validation if something is changed + ynh_script_progression --message="Validating the new configuration..." --weight=1 + + for short_setting in "${!old[@]}"; do + [[ "${changed[$short_setting]}" == "false" ]] && continue + local result="" + if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then + result="$(validate__$short_setting)" + elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + "validate__${bind%%(*}" $short_setting + fi + if [ -n "$result" ]; then + # + # Return a yaml such as: + # + # validation_errors: + # some_key: "An error message" + # some_other_key: "Another error message" + # + # We use changes_validated to know if this is + # the first validation error + if [[ "$changes_validated" == true ]]; then + ynh_return "validation_errors:" + fi + ynh_return " ${short_setting}: \"$result\"" + changes_validated=false + fi + done + + # If validation failed, exit the script right now (instead of going into apply) + # Yunohost core will pick up the errors returned via ynh_return previously + if [[ "$changes_validated" == "false" ]]; then + exit 0 + fi + +} + +ynh_app_config_get_one() { + _ynh_app_config_get_one $1 $2 $3 +} + +ynh_app_config_get() { + _ynh_app_config_get +} + +ynh_app_config_show() { + _ynh_app_config_show +} + +ynh_app_config_validate() { + _ynh_app_config_validate +} + +ynh_app_config_apply_one() { + _ynh_app_config_apply_one $1 +} +ynh_app_config_apply() { + _ynh_app_config_apply +} + +ynh_app_action_run() { + local runner="run__$1" + # Get value from getter if exists + if type -t "$runner" 2>/dev/null | grep -q '^function$' 2>/dev/null; then + $runner + #ynh_return "result:" + #ynh_return "$(echo "${result}" | sed 's/^/ /g')" + else + ynh_die "No handler defined in app's script for action $1. If you are the maintainer of this app, you should define '$runner'" + fi +} + +ynh_app_config_run() { + declare -Ag old=() + declare -Ag changed=() + declare -Ag file_hash=() + declare -Ag binds=() + declare -Ag types=() + declare -Ag formats=() + + case $1 in + show) + ynh_app_config_get + ynh_app_config_show + ;; + apply) + max_progression=4 + ynh_script_progression --message="Reading config panel description and current configuration..." + ynh_app_config_get + + ynh_app_config_validate + + ynh_script_progression --message="Applying the new configuration..." + ynh_app_config_apply + ynh_script_progression --message="Configuration of $app completed" --last + ;; + *) + ynh_app_action_run $1 + esac +} diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban new file mode 100644 index 000000000..613dcc490 --- /dev/null +++ b/helpers/helpers.v2.1.d/fail2ban @@ -0,0 +1,136 @@ +#!/bin/bash + +# Create a dedicated fail2ban config (jail and filter conf files) +# +# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# +# usage 2: ynh_add_fail2ban_config --use_template +# | arg: -t, --use_template - Use this helper in template mode +# +# This will use a template in `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf` +# See the documentation of `ynh_add_config` for a description of the template +# format and how placeholders are replaced with actual variables. +# +# Generally your template will look like that by example (for synapse): +# ``` +# f2b_jail.conf: +# [__APP__] +# enabled = true +# port = http,https +# filter = __APP__ +# logpath = /var/log/__APP__/logfile.log +# maxretry = 3 +# ``` +# ``` +# f2b_filter.conf: +# [INCLUDES] +# before = common.conf +# [Definition] +# +# # Part of regex definition (just used to make more easy to make the global regex) +# __synapse_start_line = .? \- synapse\..+ \- +# +# # Regex definition. +# failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ +# +# ignoreregex = +# ``` +# +# ----------------------------------------------------------------------------- +# +# Note about the "failregex" option: +# +# regex to match the password failure messages in the logfile. The host must be +# matched by a group named "`host`". The tag "``" can be used for standard +# IP/hostname matching and is only an alias for `(?:::f{4,6}:)?(?P[\w\-.^_]+)` +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# To validate your regex you can test with this command: +# ``` +# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf +# ``` +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_fail2ban_config() { + # Declare an array to define the options of this helper. + local legacy_args=lrmptv + local -A args_array=([l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template) + local logpath + local failregex + local max_retry + local ports + local use_template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + max_retry=${max_retry:-3} + ports=${ports:-http,https} + use_template="${use_template:-0}" + + if [ "$use_template" -ne 1 ]; then + # Usage 1, no template. Build a config file from scratch. + test -n "$logpath" || ynh_die --message="ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die --message="ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + echo " +[__APP__] +enabled = true +port = __PORTS__ +filter = __APP__ +logpath = __LOGPATH__ +maxretry = __MAX_RETRY__ +" >"$YNH_APP_BASEDIR/conf/f2b_jail.conf" + + echo " +[INCLUDES] +before = common.conf +[Definition] +failregex = __FAILREGEX__ +ignoreregex = +" >"$YNH_APP_BASEDIR/conf/f2b_filter.conf" + fi + + ynh_add_config --template="f2b_jail.conf" --destination="/etc/fail2ban/jail.d/$app.conf" + ynh_add_config --template="f2b_filter.conf" --destination="/etc/fail2ban/filter.d/$app.conf" + + # if "$logpath" doesn't exist (as if using --use_template argument), assign + # "$logpath" using the one in the previously generated fail2ban conf file + if [ -z "${logpath:-}" ]; then + # the first sed deletes possibles spaces and the second one extract the path + logpath=$(grep "^logpath" "/etc/fail2ban/jail.d/$app.conf" | sed "s/ //g" | sed "s/logpath=//g") + fi + + # Create the folder and logfile if they doesn't exist, + # as fail2ban require an existing logfile before configuration + mkdir -p "/var/log/$app" + if [ ! -f "$logpath" ]; then + touch "$logpath" + fi + # Make sure log folder's permissions are correct + chown -R "$app:$app" "/var/log/$app" + chmod -R u=rwX,g=rX,o= "/var/log/$app" + + ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + + local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" + if [[ -n "$fail2ban_error" ]]; then + ynh_print_err --message="Fail2ban failed to load the jail for $app" + ynh_print_warn --message="${fail2ban_error#*WARNING}" + fi +} + +# Remove the dedicated fail2ban config (jail and filter conf files) +# +# usage: ynh_remove_fail2ban_config +# +# Requires YunoHost version 3.5.0 or higher. +ynh_remove_fail2ban_config() { + ynh_secure_remove --file="/etc/fail2ban/jail.d/$app.conf" + ynh_secure_remove --file="/etc/fail2ban/filter.d/$app.conf" + ynh_systemd_action --service_name=fail2ban --action=reload +} diff --git a/helpers/helpers.v2.1.d/getopts b/helpers/helpers.v2.1.d/getopts new file mode 100644 index 000000000..f9ef5dc0b --- /dev/null +++ b/helpers/helpers.v2.1.d/getopts @@ -0,0 +1,215 @@ +#!/bin/bash + +# Internal helper design to allow helpers to use getopts to manage their arguments +# +# [internal] +# +# example: function my_helper() +# { +# local -A args_array=( [a]=arg1= [b]=arg2= [c]=arg3 ) +# local arg1 +# local arg2 +# local arg3 +# ynh_handle_getopts_args "$@" +# +# [...] +# } +# my_helper --arg1 "val1" -b val2 -c +# +# usage: ynh_handle_getopts_args "$@" +# | arg: $@ - Simply "$@" to tranfert all the positionnal arguments to the function +# +# This helper need an array, named "args_array" with all the arguments used by the helper +# that want to use ynh_handle_getopts_args +# Be carreful, this array has to be an associative array, as the following example: +# local -A args_array=( [a]=arg1 [b]=arg2= [c]=arg3 ) +# Let's explain this array: +# a, b and c are short options, -a, -b and -c +# arg1, arg2 and arg3 are the long options associated to the previous short ones. --arg1, --arg2 and --arg3 +# For each option, a short and long version has to be defined. +# Let's see something more significant +# local -A args_array=( [u]=user [f]=finalpath= [d]=database ) +# +# NB: Because we're using 'declare' without -g, the array will be declared as a local variable. +# +# Please keep in mind that the long option will be used as a variable to store the values for this option. +# For the previous example, that means that $finalpath will be fill with the value given as argument for this option. +# +# Also, in the previous example, finalpath has a '=' at the end. That means this option need a value. +# So, the helper has to be call with --finalpath /final/path, --finalpath=/final/path or -f /final/path, the variable $finalpath will get the value /final/path +# If there's many values for an option, -f /final /path, the value will be separated by a ';' $finalpath=/final;/path +# For an option without value, like --user in the example, the helper can be called only with --user or -u. $user will then get the value 1. +# +# To keep a retrocompatibility, a package can still call a helper, using getopts, with positional arguments. +# The "legacy mode" will manage the positional arguments and fill the variable in the same order than they are given in $args_array. +# e.g. for `my_helper "val1" val2`, arg1 will be filled with val1, and arg2 with val2. +# +# Requires YunoHost version 3.2.2 or higher. +ynh_handle_getopts_args() { + # Manage arguments only if there's some provided + set +o xtrace # set +x + if [ $# -ne 0 ]; then + # Store arguments in an array to keep each argument separated + local arguments=("$@") + + # For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u) + # And built parameters string for getopts + # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value) + local getopts_parameters="" + local option_flag="" + for option_flag in "${!args_array[@]}"; do + # Concatenate each option_flags of the array to build the string of arguments for getopts + # Will looks like 'abcd' for -a -b -c -d + # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob) + # Check the last character of the value associate to the option_flag + if [ "${args_array[$option_flag]: -1}" = "=" ]; then + # For an option with additionnal values, add a ':' after the letter for getopts. + getopts_parameters="${getopts_parameters}${option_flag}:" + else + getopts_parameters="${getopts_parameters}${option_flag}" + fi + # Check each argument given to the function + local arg="" + # ${#arguments[@]} is the size of the array + for arg in $(seq 0 $((${#arguments[@]} - 1))); do + # Escape options' values starting with -. Otherwise the - will be considered as another option. + arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}" + # And replace long option (value of the option_flag) by the short option, the option_flag itself + # (e.g. for [u]=user, --user will be -u) + # Replace long option with = (match the beginning of the argument) + arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")" + # And long option without = (match the whole line) + arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")" + done + done + + # Read and parse all the arguments + # Use a function here, to use standart arguments $@ and be able to use shift. + parse_arg() { + # Read all arguments, until no arguments are left + while [ $# -ne 0 ]; do + # Initialize the index of getopts + OPTIND=1 + # Parse with getopts only if the argument begin by -, that means the argument is an option + # getopts will fill $parameter with the letter of the option it has read. + local parameter="" + getopts ":$getopts_parameters" parameter || true + + if [ "$parameter" = "?" ]; then + ynh_die --message="Invalid argument: -${OPTARG:-}" + elif [ "$parameter" = ":" ]; then + ynh_die --message="-$OPTARG parameter requires an argument." + else + local shift_value=1 + # Use the long option, corresponding to the short option read by getopts, as a variable + # (e.g. for [u]=user, 'user' will be used as a variable) + # Also, remove '=' at the end of the long option + # The variable name will be stored in 'option_var' + local option_var="${args_array[$parameter]%=}" + # If this option doesn't take values + # if there's a '=' at the end of the long option name, this option takes values + if [ "${args_array[$parameter]: -1}" != "=" ]; then + # 'eval ${option_var}' will use the content of 'option_var' + eval ${option_var}=1 + else + # Read all other arguments to find multiple value for this option. + # Load args in a array + local all_args=("$@") + + # If the first argument is longer than 2 characters, + # There's a value attached to the option, in the same array cell + if [ ${#all_args[0]} -gt 2 ]; then + # Remove the option and the space, so keep only the value itself. + all_args[0]="${all_args[0]#-${parameter} }" + + # At this point, if all_args[0] start with "-", then the argument is not well formed + if [ "${all_args[0]:0:1}" == "-" ]; then + ynh_die --message="Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?" + fi + # Reduce the value of shift, because the option has been removed manually + shift_value=$((shift_value - 1)) + fi + + # Declare the content of option_var as a variable. + eval ${option_var}="" + # Then read the array value per value + local i + for i in $(seq 0 $((${#all_args[@]} - 1))); do + # If this argument is an option, end here. + if [ "${all_args[$i]:0:1}" == "-" ]; then + # Ignore the first value of the array, which is the option itself + if [ "$i" -ne 0 ]; then + break + fi + else + # Ignore empty parameters + if [ -n "${all_args[$i]}" ]; then + # Else, add this value to this option + # Each value will be separated by ';' + if [ -n "${!option_var}" ]; then + # If there's already another value for this option, add a ; before adding the new value + eval ${option_var}+="\;" + fi + + # Remove the \ that escape - at beginning of values. + all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}" + + # For the record. + # We're using eval here to get the content of the variable stored itself as simple text in $option_var... + # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var} + # But... ${!option_var} can't be used as left part of an assignation. + # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself. + # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one! + + eval ${option_var}+='"${all_args[$i]}"' + fi + shift_value=$((shift_value + 1)) + fi + done + fi + fi + + # Shift the parameter and its argument(s) + shift $shift_value + done + } + + # LEGACY MODE + # Check if there's getopts arguments + if [ "${arguments[0]:0:1}" != "-" ]; then + # If not, enter in legacy mode and manage the arguments as positionnal ones.. + # Dot not echo, to prevent to go through a helper output. But print only in the log. + set -x + echo "! Helper used in legacy mode !" >/dev/null + set +x + local i + for i in $(seq 0 $((${#arguments[@]} - 1))); do + # Try to use legacy_args as a list of option_flag of the array args_array + # Otherwise, fallback to getopts_parameters to get the option_flag. But an associative arrays isn't always sorted in the correct order... + # Remove all ':' in getopts_parameters + getopts_parameters=${legacy_args:-${getopts_parameters//:/}} + # Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument. + option_flag=${getopts_parameters:$i:1} + if [ -z "$option_flag" ]; then + ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored." + continue + fi + # Use the long option, corresponding to the option_flag, as a variable + # (e.g. for [u]=user, 'user' will be used as a variable) + # Also, remove '=' at the end of the long option + # The variable name will be stored in 'option_var' + local option_var="${args_array[$option_flag]%=}" + + # Store each value given as argument in the corresponding variable + # The values will be stored in the same order than $args_array + eval ${option_var}+='"${arguments[$i]}"' + done + unset legacy_args + else + # END LEGACY MODE + # Call parse_arg and pass the modified list of args as an array of arguments. + parse_arg "${arguments[@]}" + fi + fi + set -o xtrace # set -x +} diff --git a/helpers/helpers.v2.1.d/go b/helpers/helpers.v2.1.d/go new file mode 100644 index 000000000..0e18301f7 --- /dev/null +++ b/helpers/helpers.v2.1.d/go @@ -0,0 +1,247 @@ +#!/bin/bash + +ynh_go_try_bash_extension() { + if [ -x src/configure ]; then + src/configure && make -C src || { + ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." + } + fi +} + +goenv_install_dir="/opt/goenv" +go_version_path="$goenv_install_dir/versions" +# goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable. +export GOENV_ROOT="$goenv_install_dir" + +# Load the version of Go for an app, and set variables. +# +# ynh_use_go has to be used in any app scripts before using Go for the first time. +# This helper will provide alias and variables to use in your scripts. +# +# To use gem or Go, use the alias `ynh_gem` and `ynh_go` +# Those alias will use the correct version installed for the app +# For example: use `ynh_gem install` instead of `gem install` +# +# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_go` +# And propagate $PATH to sudo with $ynh_go_load_path +# Exemple: `ynh_exec_as $app $ynh_go_load_path $ynh_gem install` +# +# $PATH contains the path of the requested version of Go. +# However, $PATH is duplicated into $go_path to outlast any manipulation of $PATH +# You can use the variable `$ynh_go_load_path` to quickly load your Go version +# in $PATH for an usage into a separate script. +# Exemple: `$ynh_go_load_path $install_dir/script_that_use_gem.sh` +# +# +# Finally, to start a Go service with the correct version, 2 solutions +# Either the app is dependent of Go or gem, but does not called it directly. +# In such situation, you need to load PATH +# `Environment="__YNH_GO_LOAD_PATH__"` +# `ExecStart=__INSTALL_DIR__/my_app` +# You will replace __YNH_GO_LOAD_PATH__ with $ynh_go_load_path +# +# Or Go start the app directly, then you don't need to load the PATH variable +# `ExecStart=__YNH_GO__ my_app run` +# You will replace __YNH_GO__ with $ynh_go +# +# +# one other variable is also available +# - $go_path: The absolute path to Go binaries for the chosen version. +# +# usage: ynh_use_go +# +# Requires YunoHost version 3.2.2 or higher. +ynh_use_go () { + go_version=$(ynh_app_setting_get --app=$app --key=go_version) + + # Get the absolute path of this version of Go + go_path="$go_version_path/$go_version/bin" + + # Allow alias to be used into bash script + shopt -s expand_aliases + + # Create an alias for the specific version of Go and a variable as fallback + ynh_go="$go_path/go" + alias ynh_go="$ynh_go" + + # Load the path of this version of Go in $PATH + if [[ :$PATH: != *":$go_path"* ]]; then + PATH="$go_path:$PATH" + fi + # Create an alias to easily load the PATH + ynh_go_load_path="PATH=$PATH" + + # Sets the local application-specific Go version + pushd $install_dir + $goenv_install_dir/bin/goenv local $go_version + popd +} + +# Install a specific version of Go +# +# ynh_install_go will install the version of Go provided as argument by using goenv. +# +# This helper creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv +# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) +# +# Don't forget to execute go-dependent command in a login environment +# (e.g. sudo --login option) +# When not possible (e.g. in systemd service definition), please use direct path +# to goenv shims (e.g. $goenv_ROOT/shims/bundle) +# +# usage: ynh_install_go --go_version=go_version +# | arg: -v, --go_version= - Version of go to install. +# +# Requires YunoHost version 3.2.2 or higher. +ynh_install_go () { + # Declare an array to define the options of this helper. + local legacy_args=v + local -A args_array=( [v]=go_version= ) + local go_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Load goenv path in PATH + local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Go prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Move an existing Go binary, to avoid to block goenv + test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv + + # Install or update goenv + goenv="$(command -v goenv $goenv_install_dir/bin/goenv | head -1)" + if [ -n "$goenv" ]; then + ynh_print_info --message="goenv already seems installed in \`$goenv'." + pushd "${goenv%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/syndbg/goenv.git"; then + echo "Trying to update with Git..." + git pull -q --tags origin master + cd .. + ynh_go_try_bash_extension + fi + popd + else + ynh_print_info --message="Installing goenv with Git..." + mkdir -p $goenv_install_dir + pushd $goenv_install_dir + git init -q + git remote add -f -t master origin https://github.com/syndbg/goenv.git > /dev/null 2>&1 + git checkout -q -b master origin/master + ynh_go_try_bash_extension + goenv=$goenv_install_dir/bin/goenv + popd + fi + + goenv_latest="$(command -v "$goenv_install_dir"/plugins/*/bin/goenv-latest goenv-latest | head -1)" + if [ -n "$goenv_latest" ]; then + ynh_print_info --message="\`goenv latest' command already available in \`$goenv_latest'." + pushd "${goenv_latest%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then + ynh_print_info --message="Trying to update xxenv-latest with git..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing xxenv-latest with Git..." + mkdir -p "${goenv_install_dir}/plugins" + git clone -q https://github.com/momo-lab/xxenv-latest.git "${goenv_install_dir}/plugins/xxenv-latest" + fi + + # Enable caching + mkdir -p "${goenv_install_dir}/cache" + + # Create shims directory if needed + mkdir -p "${goenv_install_dir}/shims" + + # Restore /usr/local/bin in PATH + PATH=$CLEAR_PATH + + # And replace the old Go binary + test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go + + # Install the requested version of Go + local final_go_version=$(goenv latest --print $go_version) + ynh_print_info --message="Installation of Go-$final_go_version" + goenv install --skip-existing $final_go_version + + # Store go_version into the config of this app + ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=go_version --value=$final_go_version + + # Cleanup Go versions + ynh_cleanup_go + + # Set environment for Go users + echo "#goenv +export GOENV_ROOT=$goenv_install_dir +export PATH=\"$goenv_install_dir/bin:$PATH\" +eval \"\$(goenv init -)\" +#goenv" > /etc/profile.d/goenv.sh + + # Load the environment + eval "$(goenv init -)" +} + +# Remove the version of Go used by the app. +# +# This helper will also cleanup Go versions +# +# usage: ynh_remove_go +ynh_remove_go () { + local go_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=go_version) + + # Load goenv path in PATH + local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Go prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Remove the line for this app + ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=go_version + + # Cleanup Go versions + ynh_cleanup_go +} + +# Remove no more needed versions of Go used by the app. +# +# This helper will check what Go version are no more required, +# and uninstall them +# If no app uses Go, goenv will be also removed. +# +# usage: ynh_cleanup_go +ynh_cleanup_go () { + + # List required Go versions + local installed_apps=$(yunohost app list --output-as json --quiet | jq -r .apps[].id) + local required_go_versions="" + for installed_app in $installed_apps + do + local installed_app_go_version=$(ynh_app_setting_get --app=$installed_app --key="go_version") + if [[ $installed_app_go_version ]] + then + required_go_versions="${installed_app_go_version}\n${required_go_versions}" + fi + done + + # Remove no more needed Go versions + local installed_go_versions=$(goenv versions --bare --skip-aliases | grep -Ev '/') + for installed_go_version in $installed_go_versions + do + if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1` + then + ynh_print_info --message="Removing of Go-$installed_go_version" + $goenv_install_dir/bin/goenv uninstall --force $installed_go_version + fi + done + + # If none Go version is required + if [[ ! $required_go_versions ]] + then + # Remove goenv environment configuration + ynh_print_info --message="Removing of goenv" + ynh_secure_remove --file="$goenv_install_dir" + ynh_secure_remove --file="/etc/profile.d/goenv.sh" + fi +} diff --git a/helpers/helpers.v2.1.d/hardware b/helpers/helpers.v2.1.d/hardware new file mode 100644 index 000000000..091f023f6 --- /dev/null +++ b/helpers/helpers.v2.1.d/hardware @@ -0,0 +1,105 @@ +#!/bin/bash + +# Get the total or free amount of RAM+swap on the system +# +# [packagingv1] +# +# usage: ynh_get_ram [--free|--total] [--ignore_swap|--only_swap] +# | arg: -f, --free - Count free RAM+swap +# | arg: -t, --total - Count total RAM+swap +# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM +# | arg: -o, --only_swap - Ignore real RAM, consider only swap +# | ret: the amount of free ram, in MB (MegaBytes) +# +# Requires YunoHost version 3.8.1 or higher. +ynh_get_ram() { + # Declare an array to define the options of this helper. + local legacy_args=ftso + local -A args_array=([f]=free [t]=total [s]=ignore_swap [o]=only_swap) + local free + local total + local ignore_swap + local only_swap + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + ignore_swap=${ignore_swap:-0} + only_swap=${only_swap:-0} + free=${free:-0} + total=${total:-0} + + if [ $free -eq $total ]; then + ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram" + ram=0 + # Use the total amount of ram + elif [ $free -eq 1 ]; then + local free_ram=$(LC_ALL=C vmstat --stats --unit M | grep "free memory" | awk '{print $1}') + local free_swap=$(LC_ALL=C vmstat --stats --unit M | grep "free swap" | awk '{print $1}') + local free_ram_swap=$((free_ram + free_swap)) + + # Use the total amount of free ram + local ram=$free_ram_swap + if [ $ignore_swap -eq 1 ]; then + # Use only the amount of free ram + ram=$free_ram + elif [ $only_swap -eq 1 ]; then + # Use only the amount of free swap + ram=$free_swap + fi + elif [ $total -eq 1 ]; then + local total_ram=$(LC_ALL=C vmstat --stats --unit M | grep "total memory" | awk '{print $1}') + local total_swap=$(LC_ALL=C vmstat --stats --unit M | grep "total swap" | awk '{print $1}') + local total_ram_swap=$((total_ram + total_swap)) + + local ram=$total_ram_swap + if [ $ignore_swap -eq 1 ]; then + # Use only the amount of free ram + ram=$total_ram + elif [ $only_swap -eq 1 ]; then + # Use only the amount of free swap + ram=$total_swap + fi + fi + + echo $ram +} + +# Return 0 or 1 depending if the system has a given amount of RAM+swap free or total +# +# [packagingv1] +# +# usage: ynh_require_ram --required=RAM [--free|--total] [--ignore_swap|--only_swap] +# | arg: -r, --required= - The amount to require, in MB +# | arg: -f, --free - Count free RAM+swap +# | arg: -t, --total - Count total RAM+swap +# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM +# | arg: -o, --only_swap - Ignore real RAM, consider only swap +# | ret: 1 if the ram is under the requirement, 0 otherwise. +# +# Requires YunoHost version 3.8.1 or higher. +ynh_require_ram() { + # Declare an array to define the options of this helper. + local legacy_args=rftso + local -A args_array=([r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap) + local required + local free + local total + local ignore_swap + local only_swap + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Dunno if that's the right way to do, but that's some black magic to be able to + # forward the bool args to ynh_get_ram easily? + # If the variable $free is not empty, set it to '--free' + free=${free:+--free} + total=${total:+--total} + ignore_swap=${ignore_swap:+--ignore_swap} + only_swap=${only_swap:+--only_swap} + + local ram=$(ynh_get_ram $free $total $ignore_swap $only_swap) + + if [ $ram -lt $required ]; then + return 1 + else + return 0 + fi +} diff --git a/helpers/helpers.v2.1.d/logging b/helpers/helpers.v2.1.d/logging new file mode 100644 index 000000000..accb8f9b0 --- /dev/null +++ b/helpers/helpers.v2.1.d/logging @@ -0,0 +1,347 @@ +#!/bin/bash + +# Print a message to stderr and exit +# +# usage: ynh_die --message=MSG [--ret_code=RETCODE] +# | arg: -m, --message= - Message to display +# | arg: -c, --ret_code= - Exit code to exit with +# +# Requires YunoHost version 2.4.0 or higher. +ynh_die() { + # Declare an array to define the options of this helper. + local legacy_args=mc + local -A args_array=([m]=message= [c]=ret_code=) + local message + local ret_code + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + ret_code=${ret_code:-1} + + echo "$message" 1>&2 + exit "$ret_code" +} + +# Display a message in the 'INFO' logging category +# +# usage: ynh_print_info --message="Some message" +# | arg: -m, --message= - Message to display +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_info() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=message=) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + echo "$message" >&$YNH_STDINFO +} + +# Main printer, just in case in the future we have to change anything about that. +# +# [internal] +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_log() { + echo -e "${1}" +} + +# Print a warning on stderr +# +# usage: ynh_print_warn --message="Text to print" +# | arg: -m, --message= - The text to print +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_warn() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=message=) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_print_log "${message}" >&2 +} + +# Print an error on stderr +# +# usage: ynh_print_err --message="Text to print" +# | arg: -m, --message= - The text to print +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_err() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=message=) + local message + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_print_log "[Error] ${message}" >&2 +} + +# Execute a command and print the result as an error +# +# usage: ynh_exec_err your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_err +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_err() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + ynh_print_err --message="$(eval $@)" + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + ynh_print_err --message="$("$@")" + fi +} + +# Execute a command and print the result as a warning +# +# usage: ynh_exec_warn your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_warn +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_warn() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + ynh_print_warn --message="$(eval $@)" + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + ynh_print_warn --message="$("$@")" + fi +} + +# Execute a command and force the result to be printed on stdout +# +# usage: ynh_exec_warn_less your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_warn +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_warn_less() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + eval $@ 2>&1 + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" 2>&1 + fi +} + +# Execute a command and redirect stdout in /dev/null +# +# usage: ynh_exec_quiet your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_warn +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_quiet() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + eval $@ > /dev/null + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" > /dev/null + fi +} + +# Execute a command and redirect stdout and stderr in /dev/null +# +# usage: ynh_exec_quiet your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_quiet +# +# Requires YunoHost version 3.2.0 or higher. +ynh_exec_fully_quiet() { + # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes, + # (because in the past eval was used) ... + # we detect this by checking that there's no 2nd arg, and $1 contains a space + if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]] + then + eval $@ > /dev/null 2>&1 + else + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" > /dev/null 2>&1 + fi +} + +# Execute a command and redirect stderr in /dev/null. Print stderr on error. +# +# usage: ynh_exec_and_print_stderr_only_if_error your command and args +# | arg: command - command to execute +# +# Note that you should NOT quote the command but only prefix it with ynh_exec_and_print_stderr_only_if_error +# +# Requires YunoHost version 11.2 or higher. +ynh_exec_and_print_stderr_only_if_error() { + logfile="$(mktemp)" + rc=0 + # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077 + "$@" 2> "$logfile" || rc="$?" + if (( rc != 0 )); then + ynh_exec_warn cat "$logfile" + ynh_secure_remove "$logfile" + return "$rc" + fi +} + +# Remove any logs for all the following commands. +# +# usage: ynh_print_OFF +# +# [internal] +# +# WARNING: You should be careful with this helper, and never forget to use ynh_print_ON as soon as possible to restore the logging. +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_OFF() { + exec {BASH_XTRACEFD}>/dev/null +} + +# Restore the logging after ynh_print_OFF +# +# usage: ynh_print_ON +# +# [internal] +# +# Requires YunoHost version 3.2.0 or higher. +ynh_print_ON() { + exec {BASH_XTRACEFD}>&1 + # Print an echo only for the log, to be able to know that ynh_print_ON has been called. + echo ynh_print_ON >/dev/null +} + +# Initial definitions for ynh_script_progression +increment_progression=0 +previous_weight=0 +max_progression=-1 +# Set the scale of the progression bar +# progress_string(0,1,2) should have the size of the scale. +progress_scale=20 +progress_string2="####################" +progress_string1="++++++++++++++++++++" +progress_string0="...................." +# Define base_time when the file is sourced +base_time=$(date +%s) + +# Print a progress bar showing the progression of an app script +# +# usage: ynh_script_progression --message=message [--weight=weight] [--time] +# | arg: -m, --message= - The text to print +# | arg: -w, --weight= - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script. +# | arg: -t, --time - Print the execution time since the last call to this helper. Especially usefull to define weights. The execution time is given for the duration since the previous call. So the weight should be applied to this previous call. +# | arg: -l, --last - Use for the last call of the helper, to fill the progression bar. +# +# Requires YunoHost version 3.5.0 or higher. +ynh_script_progression() { + set +o xtrace # set +x + # Declare an array to define the options of this helper. + local legacy_args=mwtl + local -A args_array=([m]=message= [w]=weight= [t]=time [l]=last) + local message + local weight + local time + local last + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Re-disable xtrace, ynh_handle_getopts_args set it back + set +o xtrace # set +x + weight=${weight:-1} + + # Always activate time when running inside CI tests + if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then + time=${time:-1} + else + time=${time:-0} + fi + + last=${last:-0} + + # Get execution time since the last $base_time + local exec_time=$(($(date +%s) - $base_time)) + base_time=$(date +%s) + + # Compute $max_progression (if we didn't already) + if [ "$max_progression" = -1 ]; then + # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented. + local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)" + # Get the number of call with a weight value + local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0) + + # Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight + local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]]*\).*/\1/g')" + # Get the weight of each occurrences of 'ynh_script_progression' in the script using -w + local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]]*\).*/\1/g')" + # Each value will be on a different line. + # Remove each 'end of line' and replace it by a '+' to sum the values. + local weight_values=$(($(echo "$weight_valuesA" "$weight_valuesB" | grep -v -E '^\s*$' | tr '\n' '+' | sed 's/+$/+0/g'))) + + # max_progression is a total number of calls to this helper. + # Less the number of calls with a weight value. + # Plus the total of weight values + max_progression=$(($helper_calls - $weight_calls + $weight_values)) + fi + + # Increment each execution of ynh_script_progression in this script by the weight of the previous call. + increment_progression=$(($increment_progression + $previous_weight)) + # Store the weight of the current call in $previous_weight for next call + previous_weight=$weight + + # Reduce $increment_progression to the size of the scale + if [ $last -eq 0 ]; then + local effective_progression=$(($increment_progression * $progress_scale / $max_progression)) + # If last is specified, fill immediately the progression_bar + else + local effective_progression=$progress_scale + fi + + # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task + # expected_progression is the progression expected after the current task + local expected_progression="$((($increment_progression + $weight) * $progress_scale / $max_progression - $effective_progression))" + if [ $last -eq 1 ]; then + expected_progression=0 + fi + # left_progression is the progression not yet done + local left_progression="$(($progress_scale - $effective_progression - $expected_progression))" + # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done. + local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}" + + local print_exec_time="" + if [ $time -eq 1 ] && [ "$exec_time" -gt 10 ]; then + print_exec_time=" [$(bc <<< "scale=1; $exec_time / 60" ) minutes]" + fi + + ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" + set -o xtrace # set -x +} + +# Return data to the YunoHost core for later processing +# (to be used by special hooks like app config panel and core diagnosis) +# +# usage: ynh_return somedata +# +# Requires YunoHost version 3.6.0 or higher. +ynh_return() { + echo "$1" >>"$YNH_STDRETURN" +} diff --git a/helpers/helpers.v2.1.d/logrotate b/helpers/helpers.v2.1.d/logrotate new file mode 100644 index 000000000..efc1137c1 --- /dev/null +++ b/helpers/helpers.v2.1.d/logrotate @@ -0,0 +1,103 @@ +#!/bin/bash + +FIRST_CALL_TO_LOGROTATE="true" + +# Use logrotate to manage the logfile +# +# usage: ynh_use_logrotate [--logfile=/log/file] [--specific_user=user/group] +# | arg: -l, --logfile= - absolute path of logfile +# | arg: -u, --specific_user= - run logrotate as the specified user and group. If not specified logrotate is runned as root. +# +# If no `--logfile` is provided, `/var/log/$app` will be used as default. +# `logfile` can point to a directory or a file. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_use_logrotate() { + + # Stupid patch to ignore legacy --non-append and --nonappend + # which was never properly understood and improperly used and kind of bullshit + local all_args=( ${@} ) + for I in $(seq 0 $(($# - 1))) + do + if [[ "${all_args[$I]}" == "--non-append" ]] || [[ "${all_args[$I]}" == "--nonappend" ]] + then + unset all_args[$I] + fi + done + set -- "${all_args[@]}" + + # Argument parsing + local legacy_args=lu + local -A args_array=([l]=logfile= [u]=specific_user=) + local logfile + local specific_user + ynh_handle_getopts_args "$@" + logfile="${logfile:-}" + specific_user="${specific_user:-}" + + set -o noglob + if [[ -z "$logfile" ]]; then + logfile="/var/log/${app}/*.log" + elif [[ "${logfile##*.}" != "log" ]] && [[ "${logfile##*.}" != "txt" ]]; then + logfile="$logfile/*.log" + fi + set +o noglob + + for stuff in $logfile + do + mkdir --parents $(dirname "$stuff") + done + + local su_directive="" + if [[ -n "$specific_user" ]]; then + su_directive="su ${specific_user%/*} ${specific_user#*/}" + fi + + local tempconf="$(mktemp)" + cat << EOF >$tempconf +$logfile { + # Rotate if the logfile exceeds 100Mo + size 100M + # Keep 12 old log maximum + rotate 12 + # Compress the logs with gzip + compress + # Compress the log at the next cycle. So keep always 2 non compressed logs + delaycompress + # Copy and truncate the log to allow to continue write on it. Instead of moving the log. + copytruncate + # Do not trigger an error if the log is missing + missingok + # Do not rotate if the log is empty + notifempty + # Keep old logs in the same dir + noolddir + $su_directive +} +EOF + + if [[ "$FIRST_CALL_TO_LOGROTATE" == "true" ]] + then + cat $tempconf > /etc/logrotate.d/$app + else + cat $tempconf >> /etc/logrotate.d/$app + fi + + FIRST_CALL_TO_LOGROTATE="false" + + # Make sure permissions are correct (otherwise the config file could be ignored and the corresponding logs never rotated) + chmod 644 "/etc/logrotate.d/$app" + mkdir -p "/var/log/$app" + chmod 750 "/var/log/$app" +} + +# Remove the app's logrotate config. +# +# usage: ynh_remove_logrotate +# +# Requires YunoHost version 2.6.4 or higher. +ynh_remove_logrotate() { + if [ -e "/etc/logrotate.d/$app" ]; then + rm "/etc/logrotate.d/$app" + fi +} diff --git a/helpers/helpers.v2.1.d/mongodb b/helpers/helpers.v2.1.d/mongodb new file mode 100644 index 000000000..d40d11bfe --- /dev/null +++ b/helpers/helpers.v2.1.d/mongodb @@ -0,0 +1,355 @@ +#!/bin/bash + +# Execute a mongo command +# +# example: ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("wekan")' +# example: ynh_mongo_exec --command="db.getMongo().getDBNames().indexOf(\"wekan\")" +# +# usage: ynh_mongo_exec [--user=user] [--password=password] [--authenticationdatabase=authenticationdatabase] [--database=database] [--host=host] [--port=port] --command="command" [--eval] +# | arg: -u, --user= - The user name to connect as +# | arg: -p, --password= - The user password +# | arg: -d, --authenticationdatabase= - The authenticationdatabase to connect to +# | arg: -d, --database= - The database to connect to +# | arg: -h, --host= - The host to connect to +# | arg: -P, --port= - The port to connect to +# | arg: -c, --command= - The command to evaluate +# | arg: -e, --eval - Evaluate instead of execute the command. +# +# +ynh_mongo_exec() { + # Declare an array to define the options of this helper. + local legacy_args=upadhPce + local -A args_array=( [u]=user= [p]=password= [a]=authenticationdatabase= [d]=database= [h]=host= [P]=port= [c]=command= [e]=eval ) + local user + local password + local authenticationdatabase + local database + local host + local port + local command + local eval + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + user="${user:-}" + password="${password:-}" + authenticationdatabase="${authenticationdatabase:-}" + database="${database:-}" + host="${host:-}" + port="${port:-}" + eval=${eval:-0} + + # If user is provided + if [ -n "$user" ] + then + user="--username=$user" + + # If password is provided + if [ -n "$password" ] + then + password="--password=$password" + fi + + # If authenticationdatabase is provided + if [ -n "$authenticationdatabase" ] + then + authenticationdatabase="--authenticationDatabase=$authenticationdatabase" + else + authenticationdatabase="--authenticationDatabase=admin" + fi + else + password="" + authenticationdatabase="" + fi + + # If host is provided + if [ -n "$host" ] + then + host="--host=$host" + fi + + # If port is provided + if [ -n "$port" ] + then + port="--port=$port" + fi + + # If eval is not provided + if [ $eval -eq 0 ] + then + # If database is provided + if [ -n "$database" ] + then + database="use $database" + else + database="" + fi + + mongosh --quiet --username $user --password $password --authenticationDatabase $authenticationdatabase --host $host --port $port < ./dump.bson +# +# usage: ynh_mongo_dump_db --database=database +# | arg: -d, --database= - The database name to dump +# | ret: the mongodump output +# +# +ynh_mongo_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=( [d]=database= ) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + mongodump --quiet --db="$database" --archive +} + +# Create a user +# +# [internal] +# +# usage: ynh_mongo_create_user --db_user=user --db_pwd=pwd --db_name=name +# | arg: -u, --db_user= - The user name to create +# | arg: -p, --db_pwd= - The password to identify user by +# | arg: -n, --db_name= - Name of the database to grant privilegies +# +# +ynh_mongo_create_user() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) + local db_user + local db_name + local db_pwd + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Create the user and set the user as admin of the db + ynh_mongo_exec --database="$db_name" --command='db.createUser( { user: "'${db_user}'", pwd: "'${db_pwd}'", roles: [ { role: "readWrite", db: "'${db_name}'" } ] } );' + + # Add clustermonitoring rights + ynh_mongo_exec --database="$db_name" --command='db.grantRolesToUser("'${db_user}'",[{ role: "clusterMonitor", db: "admin" }]);' +} + +# Check if a mongo database exists +# +# usage: ynh_mongo_database_exists --database=database +# | arg: -d, --database= - The database for which to check existence +# | exit: Return 1 if the database doesn't exist, 0 otherwise +# +# +ynh_mongo_database_exists() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [ $(ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("'${database}'")' --eval) -lt 0 ] + then + return 1 + else + return 0 + fi +} + +# Restore a database +# +# example: ynh_mongo_restore_db --database=wekan < ./dump.bson +# +# usage: ynh_mongo_restore_db --database=database +# | arg: -d, --database= - The database name to restore +# +# +ynh_mongo_restore_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=( [d]=database= ) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + mongorestore --quiet --db="$database" --archive +} + +# Drop a user +# +# [internal] +# +# usage: ynh_mongo_drop_user --db_user=user --db_name=name +# | arg: -u, --db_user= - The user to drop +# | arg: -n, --db_name= - Name of the database +# +# +ynh_mongo_drop_user() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -A args_array=( [u]=db_user= [n]=db_name= ) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_mongo_exec --database="$db_name" --command='db.dropUser("'$db_user'", {w: "majority", wtimeout: 5000})' +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# usage: ynh_mongo_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# +# After executing this helper, the password of the created database will be available in $db_pwd +# It will also be stored as "mongopwd" into the app settings. +# +# +ynh_mongo_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= ) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local new_db_pwd=$(ynh_string_random) # Generate a random password + # If $db_pwd is not provided, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + # Create the user and grant access to the database + ynh_mongo_create_user --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name" + + # Store the password in the app's config + ynh_app_setting_set --app=$app --key=db_pwd --value=$db_pwd +} + +# Remove a database if it exists, and the associated user +# +# usage: ynh_mongo_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# +# +ynh_mongo_remove_db() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -A args_array=( [u]=db_user= [n]=db_name= ) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists + ynh_mongo_drop_db --database=$db_name # Remove the database + else + ynh_print_warn --message="Database $db_name not found" + fi + + # Remove mongo user if it exists + ynh_mongo_drop_user --db_user=$db_user --db_name=$db_name +} + +# Install MongoDB and integrate MongoDB service in YunoHost +# +# usage: ynh_install_mongo [--mongo_version=mongo_version] +# | arg: -m, --mongo_version= - Version of MongoDB to install +# +# +ynh_install_mongo() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=mongo_version=) + local mongo_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + mongo_version="${mongo_version:-$YNH_MONGO_VERSION}" + + ynh_print_info --message="Installing MongoDB Community Edition ..." + local mongo_debian_release=$(ynh_get_debian_release) + + if [[ $(cat /proc/cpuinfo) != *"avx"* && "$mongo_version" != "4.4" ]]; then + ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)." + mongo_version="4.4" + fi + if [[ "$mongo_version" == "4.4" ]]; then + ynh_print_warn --message="Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release." + mongo_debian_release=buster + fi + + ynh_install_extra_app_dependencies --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc" + mongodb_servicename=mongod + + # Make sure MongoDB is started and enabled + systemctl enable $mongodb_servicename --quiet + systemctl daemon-reload --quiet + ynh_systemd_action --service_name=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log" + + # Integrate MongoDB service in YunoHost + yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log" + + # Store mongo_version into the config of this app + ynh_app_setting_set --app=$app --key=mongo_version --value=$mongo_version +} + +# Remove MongoDB +# Only remove the MongoDB service integration in YunoHost for now +# if MongoDB package as been removed +# +# usage: ynh_remove_mongo +# +# +ynh_remove_mongo() { + # Only remove the mongodb service if it is not installed. + if ! ynh_package_is_installed --package="mongodb*" + then + ynh_print_info --message="Removing MongoDB service..." + mongodb_servicename=mongod + # Remove the mongodb service + yunohost service remove $mongodb_servicename + ynh_secure_remove --file="/var/lib/mongodb" + ynh_secure_remove --file="/var/log/mongodb" + fi +} diff --git a/helpers/helpers.v2.1.d/multimedia b/helpers/helpers.v2.1.d/multimedia new file mode 100644 index 000000000..c860ae49f --- /dev/null +++ b/helpers/helpers.v2.1.d/multimedia @@ -0,0 +1,103 @@ +#!/bin/bash + +readonly MEDIA_GROUP=multimedia +readonly MEDIA_DIRECTORY=/home/yunohost.multimedia + +# Initialize the multimedia directory system +# +# usage: ynh_multimedia_build_main_dir +# +# Requires YunoHost version 4.2 or higher. +ynh_multimedia_build_main_dir() { + + ## Création du groupe multimedia + groupadd -f $MEDIA_GROUP + + ## Création des dossiers génériques + mkdir -p "$MEDIA_DIRECTORY" + mkdir -p "$MEDIA_DIRECTORY/share" + mkdir -p "$MEDIA_DIRECTORY/share/Music" + mkdir -p "$MEDIA_DIRECTORY/share/Picture" + mkdir -p "$MEDIA_DIRECTORY/share/Video" + mkdir -p "$MEDIA_DIRECTORY/share/eBook" + + ## Création des dossiers utilisateurs + for user in $(yunohost user list --output-as json | jq -r '.users | keys[]'); do + mkdir -p "$MEDIA_DIRECTORY/$user" + mkdir -p "$MEDIA_DIRECTORY/$user/Music" + mkdir -p "$MEDIA_DIRECTORY/$user/Picture" + mkdir -p "$MEDIA_DIRECTORY/$user/Video" + mkdir -p "$MEDIA_DIRECTORY/$user/eBook" + ln -sfn "$MEDIA_DIRECTORY/share" "$MEDIA_DIRECTORY/$user/Share" + # Création du lien symbolique dans le home de l'utilisateur. + #link will only be created if the home directory of the user exists and if it's located in '/home' folder + local user_home="$(getent passwd $user | cut -d: -f6 | grep '^/home/')" + if [[ -d "$user_home" ]]; then + ln -sfn "$MEDIA_DIRECTORY/$user" "$user_home/Multimedia" + fi + # Propriétaires des dossiers utilisateurs. + chown -R $user "$MEDIA_DIRECTORY/$user" + done + # Default yunohost hooks for post_user_create,delete will take care + # of creating/deleting corresponding multimedia folders when users + # are created/deleted in the future... + + ## Application des droits étendus sur le dossier multimedia. + # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other: + setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true + # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers. + setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true + # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl. + setfacl -RL -m m::rwx "$MEDIA_DIRECTORY" || true +} + +# Add a directory in yunohost.multimedia +# +# usage: ynh_multimedia_addfolder --source_dir="source_dir" --dest_dir="dest_dir" +# +# | arg: -s, --source_dir= - Source directory - The real directory which contains your medias. +# | arg: -d, --dest_dir= - Destination directory - The name and the place of the symbolic link, relative to "/home/yunohost.multimedia" +# +# This "directory" will be a symbolic link to a existing directory. +# +# Requires YunoHost version 4.2 or higher. +ynh_multimedia_addfolder() { + + # Declare an array to define the options of this helper. + local legacy_args=sd + local -A args_array=([s]=source_dir= [d]=dest_dir=) + local source_dir + local dest_dir + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Ajout d'un lien symbolique vers le dossier à partager + ln -sfn "$source_dir" "$MEDIA_DIRECTORY/$dest_dir" + + ## Application des droits étendus sur le dossier ajouté + # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other: + setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir" + # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers. + setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir" + # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl. + setfacl -RL -m m::rwx "$source_dir" +} + +# Allow an user to have an write authorisation in multimedia directories +# +# usage: ynh_multimedia_addaccess user_name +# +# | arg: -u, --user_name= - The name of the user which gain this access. +# +# Requires YunoHost version 4.2 or higher. +ynh_multimedia_addaccess() { + # Declare an array to define the options of this helper. + local legacy_args=u + declare -Ar args_array=([u]=user_name=) + local user_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + groupadd -f multimedia + usermod -a -G multimedia $user_name +} diff --git a/helpers/helpers.v2.1.d/mysql b/helpers/helpers.v2.1.d/mysql new file mode 100644 index 000000000..c11f7989a --- /dev/null +++ b/helpers/helpers.v2.1.d/mysql @@ -0,0 +1,249 @@ +#!/bin/bash + +# Open a connection as a user +# +# usage: ynh_mysql_connect_as --user=user --password=password [--database=database] +# | arg: -u, --user= - the user name to connect as +# | arg: -p, --password= - the user password +# | arg: -d, --database= - the database to connect to +# +# examples: +# ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;" +# ynh_mysql_connect_as --user="user" --password="pass" < /path/to/file.sql +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_connect_as() { + # Declare an array to define the options of this helper. + local legacy_args=upd + local -A args_array=([u]=user= [p]=password= [d]=database=) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + mysql --user="$user" --password="$password" --batch "$database" +} + +# Execute a command as root user +# +# usage: ynh_mysql_execute_as_root --sql=sql [--database=database] +# | arg: -s, --sql= - the SQL command to execute +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_execute_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=sd + local -A args_array=([s]=sql= [d]=database=) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + mysql -B "$database" <<<"$sql" +} + +# Execute a command from a file as root user +# +# usage: ynh_mysql_execute_file_as_root --file=file [--database=database] +# | arg: -f, --file= - the file containing SQL commands +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_execute_file_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=fd + local -A args_array=([f]=file= [d]=database=) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + mysql -B "$database" <"$file" +} + +# Create a database and grant optionnaly privilegies to a user +# +# [internal] +# +# usage: ynh_mysql_create_db db [user [pwd]] +# | arg: db - the database name to create +# | arg: user - the user to grant privilegies +# | arg: pwd - the password to identify user by +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_create_db() { + local db=$1 + + local sql="CREATE DATABASE ${db};" + + # grant all privilegies to user + if [[ $# -gt 1 ]]; then + sql+=" GRANT ALL PRIVILEGES ON ${db}.* TO '${2}'@'localhost'" + if [[ -n ${3:-} ]]; then + sql+=" IDENTIFIED BY '${3}'" + fi + sql+=" WITH GRANT OPTION;" + fi + + ynh_mysql_execute_as_root --sql="$sql" +} + +# Drop a database +# +# [internal] +# +# If you intend to drop the database *and* the associated user, +# consider using ynh_mysql_remove_db instead. +# +# usage: ynh_mysql_drop_db db +# | arg: db - the database name to drop +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_drop_db() { + ynh_mysql_execute_as_root --sql="DROP DATABASE ${1};" +} + +# Dump a database +# +# usage: ynh_mysql_dump_db --database=database +# | arg: -d, --database= - the database name to dump +# | ret: The mysqldump output +# +# example: ynh_mysql_dump_db --database=roundcube > ./dump.sql +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + mysqldump --single-transaction --skip-dump-date --routines "$database" +} + +# Create a user +# +# [internal] +# +# usage: ynh_mysql_create_user user pwd [host] +# | arg: user - the user name to create +# | arg: pwd - the password to identify user by +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_create_user() { + ynh_mysql_execute_as_root \ + --sql="CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';" +} + +# Check if a mysql user exists +# +# [internal] +# +# usage: ynh_mysql_user_exists --user=user +# | arg: -u, --user= - the user for which to check existence +# | ret: 0 if the user exists, 1 otherwise. +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=user=) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]; then + return 1 + else + return 0 + fi +} + +# Drop a user +# +# [internal] +# +# usage: ynh_mysql_drop_user user +# | arg: user - the user name to drop +# +# Requires YunoHost version 2.2.4 or higher. +ynh_mysql_drop_user() { + ynh_mysql_execute_as_root --sql="DROP USER '${1}'@'localhost';" +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# [packagingv1] +# +# usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# +# After executing this helper, the password of the created database will be available in `$db_pwd` +# It will also be stored as "`mysqlpwd`" into the app settings. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_mysql_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Generate a random password + local new_db_pwd=$(ynh_string_random) + # If $db_pwd is not provided, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" + ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd +} + +# Remove a database if it exists, and the associated user +# +# [packagingv1] +# +# usage: ynh_mysql_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# +# Requires YunoHost version 2.6.4 or higher. +ynh_mysql_remove_db() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -Ar args_array=([u]=db_user= [n]=db_name=) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if mysqlshow | grep -q "^| $db_name "; then + ynh_mysql_drop_db $db_name + else + ynh_print_warn --message="Database $db_name not found" + fi + + # Remove mysql user if it exists + if ynh_mysql_user_exists --user=$db_user; then + ynh_mysql_drop_user $db_user + fi +} diff --git a/helpers/helpers.v2.1.d/network b/helpers/helpers.v2.1.d/network new file mode 100644 index 000000000..bed9dd402 --- /dev/null +++ b/helpers/helpers.v2.1.d/network @@ -0,0 +1,132 @@ +#!/bin/bash + +# Find a free port and return it +# +# [packagingv1] +# +# usage: ynh_find_port --port=begin_port +# | arg: -p, --port= - port to start to search +# | ret: the port number +# +# example: port=$(ynh_find_port --port=8080) +# +# Requires YunoHost version 2.6.4 or higher. +ynh_find_port() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=port=) + local port + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." + while ! ynh_port_available --port=$port; do + port=$((port + 1)) + done + echo $port +} + +# Test if a port is available +# +# [packagingv1] +# +# usage: ynh_find_port --port=XYZ +# | arg: -p, --port= - port to check +# | ret: 0 if the port is available, 1 if it is already used by another process. +# +# example: ynh_port_available --port=1234 || ynh_die --message="Port 1234 is needs to be available for this app" +# +# Requires YunoHost version 3.8.0 or higher. +ynh_port_available() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=port=) + local port + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Check if the port is free + if ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ":$port$"; then + return 1 + # This is to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up) + elif grep -q "port: '$port'" /etc/yunohost/apps/*/settings.yml; then + return 1 + else + return 0 + fi +} + +# Validate an IP address +# +# [internal] +# +# usage: ynh_validate_ip --family=family --ip_address=ip_address +# | ret: 0 for valid ip addresses, 1 otherwise +# +# example: ynh_validate_ip 4 111.222.333.444 +# +# Requires YunoHost version 2.2.4 or higher. +ynh_validate_ip() { + # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 + + # Declare an array to define the options of this helper. + local legacy_args=fi + local -A args_array=([f]=family= [i]=ip_address=) + local family + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 + + python3 /dev/stdin <"$n_install_dir/node_update.sh" <"/etc/cron.daily/node_update" <> $n_install_dir/node_update.log +EOF + + chmod +x "/etc/cron.daily/node_update" +} diff --git a/helpers/helpers.v2.1.d/permission b/helpers/helpers.v2.1.d/permission new file mode 100644 index 000000000..d3eb71c22 --- /dev/null +++ b/helpers/helpers.v2.1.d/permission @@ -0,0 +1,399 @@ +#!/bin/bash + +# Create a new permission for the app +# +# Example 1: `ynh_permission_create --permission=admin --url=/admin --additional_urls=domain.tld/admin /superadmin --allowed=alice bob \ +# --label="My app admin" --show_tile=true` +# +# This example will create a new permission permission with this following effect: +# - A tile named "My app admin" in the SSO will be available for the users alice and bob. This tile will point to the relative url '/admin'. +# - Only the user alice and bob will have the access to theses following url: /admin, domain.tld/admin, /superadmin +# +# +# Example 2: +# +# ynh_permission_create --permission=api --url=domain.tld/api --auth_header=false --allowed=visitors \ +# --label="MyApp API" --protected=true +# +# This example will create a new protected permission. So the admin won't be able to add/remove the visitors group of this permission. +# In case of an API with need to be always public it avoid that the admin break anything. +# With this permission all client will be allowed to access to the url 'domain.tld/api'. +# Note that in this case no tile will be show on the SSO. +# Note that the auth_header parameter is to 'false'. So no authentication header will be passed to the application. +# Generally the API is requested by an application and enabling the auth_header has no advantage and could bring some issues in some case. +# So in this case it's better to disable this option for all API. +# +# +# usage: ynh_permission_create --permission="permission" [--url="url"] [--additional_urls="second-url" [ "third-url" ]] [--auth_header=true|false] +# [--allowed=group1 [ group2 ]] [--label="label"] [--show_tile=true|false] +# [--protected=true|false] +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if 'show_tile' is enabled, this URL will be the URL of the tile. +# | arg: -A, --additional_urls= - (optional) List of additional URL for which access will be allowed/forbidden +# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true +# | arg: -a, --allowed= - (optional) A list of group/user to allow for the permission +# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. Default is "APP_LABEL (permission name)". +# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. Defaults to false for the permission different than 'main'. +# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. +# +# [packagingv1] +# +# If provided, 'url' or 'additional_urls' is assumed to be relative to the app domain/path if they +# start with '/'. For example: +# / -> domain.tld/app +# /admin -> domain.tld/app/admin +# domain.tld/app/api -> domain.tld/app/api +# +# 'url' or 'additional_urls' can be treated as a PCRE (not lua) regex if it starts with "re:". +# For example: +# re:/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$ +# re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$ +# +# Note that globally the parameter 'url' and 'additional_urls' are same. The only difference is: +# - 'url' is only one url, 'additional_urls' can be a list of urls. There are no limitation of 'additional_urls' +# - 'url' is used for the url of tile in the SSO (if enabled with the 'show_tile' parameter) +# +# +# About the authentication header (auth_header parameter). +# The SSO pass (by default) to the application theses following HTTP header (linked to the authenticated user) to the application: +# - "Auth-User": username +# - "Remote-User": username +# - "Email": user email +# +# Generally this feature is usefull to authenticate automatically the user in the application but in some case the application don't work with theses header and theses header need to be disabled to have the application to work correctly. +# See https://github.com/YunoHost/issues/issues/1420 for more informations +# +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_create() { + # Declare an array to define the options of this helper. + local legacy_args=puAhaltP + local -A args_array=([p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected=) + local permission + local url + local additional_urls + local auth_header + local allowed + local label + local show_tile + local protected + ynh_handle_getopts_args "$@" + url=${url:-} + additional_urls=${additional_urls:-} + auth_header=${auth_header:-} + allowed=${allowed:-} + label=${label:-} + show_tile=${show_tile:-} + protected=${protected:-} + + if [[ -n $url ]]; then + url=",url='$url'" + fi + + if [[ -n $additional_urls ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # By example: + # --additional_urls /urlA /urlB + # will be: + # additional_urls=['/urlA', '/urlB'] + additional_urls=",additional_urls=['${additional_urls//;/\',\'}']" + fi + + if [[ -n $auth_header ]]; then + if [ $auth_header == "true" ]; then + auth_header=",auth_header=True" + else + auth_header=",auth_header=False" + fi + fi + + if [[ -n $allowed ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # By example: + # --allowed alice bob + # will be: + # allowed=['alice', 'bob'] + allowed=",allowed=['${allowed//;/\',\'}']" + fi + + if [[ -n ${label:-} ]]; then + label=",label='$label'" + else + label=",label='$permission'" + fi + + if [[ -n ${show_tile:-} ]]; then + if [ $show_tile == "true" ]; then + show_tile=",show_tile=True" + else + show_tile=",show_tile=False" + fi + fi + + if [[ -n ${protected:-} ]]; then + if [ $protected == "true" ]; then + protected=",protected=True" + else + protected=",protected=False" + fi + fi + + yunohost tools shell -c "from yunohost.permission import permission_create; permission_create('$app.$permission' $url $additional_urls $auth_header $allowed $label $show_tile $protected)" +} + +# Remove a permission for the app (note that when the app is removed all permission is automatically removed) +# +# [packagingv1] +# +# example: ynh_permission_delete --permission=editors +# +# usage: ynh_permission_delete --permission="permission" +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_delete() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=permission=) + local permission + ynh_handle_getopts_args "$@" + + yunohost tools shell -c "from yunohost.permission import permission_delete; permission_delete('$app.$permission')" +} + +# Check if a permission exists +# +# [packagingv1] +# +# usage: ynh_permission_exists --permission=permission +# | arg: -p, --permission= - the permission to check +# | exit: Return 1 if the permission doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_exists() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=permission=) + local permission + ynh_handle_getopts_args "$@" + + yunohost user permission list "$app" --output-as json --quiet \ + | jq -e --arg perm "$app.$permission" '.permissions[$perm]' >/dev/null +} + +# Redefine the url associated to a permission +# +# [packagingv1] +# +# usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]] +# [--auth_header=true|false] [--clear_urls] +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) +# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if you want to remove url you can pass an empty sting as arguments (""). +# | arg: -a, --add_url= - (optional) List of additional url to add for which access will be allowed/forbidden. +# | arg: -r, --remove_url= - (optional) List of additional url to remove for which access will be allowed/forbidden +# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application +# | arg: -c, --clear_urls - (optional) Clean all urls (url and additional_urls) +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_url() { + # Declare an array to define the options of this helper. + local legacy_args=puarhc + local -A args_array=([p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls) + local permission + local url + local add_url + local remove_url + local auth_header + local clear_urls + ynh_handle_getopts_args "$@" + url=${url:-} + add_url=${add_url:-} + remove_url=${remove_url:-} + auth_header=${auth_header:-} + clear_urls=${clear_urls:-} + + if [[ -n $url ]]; then + url=",url='$url'" + fi + + if [[ -n $add_url ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --add_url /urlA /urlB + # will be: + # add_url=['/urlA', '/urlB'] + add_url=",add_url=['${add_url//;/\',\'}']" + fi + + if [[ -n $remove_url ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --remove_url /urlA /urlB + # will be: + # remove_url=['/urlA', '/urlB'] + remove_url=",remove_url=['${remove_url//;/\',\'}']" + fi + + if [[ -n $auth_header ]]; then + if [ $auth_header == "true" ]; then + auth_header=",auth_header=True" + else + auth_header=",auth_header=False" + fi + fi + + if [[ -n $clear_urls ]] && [ $clear_urls -eq 1 ]; then + clear_urls=",clear_urls=True" + fi + + yunohost tools shell -c "from yunohost.permission import permission_url; permission_url('$app.$permission' $url $add_url $remove_url $auth_header $clear_urls)" +} + +# Update a permission for the app +# +# [packagingv1] +# +# usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]] +# [--label="label"] [--show_tile=true|false] [--protected=true|false] +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: -a, --add= - the list of group or users to enable add to the permission +# | arg: -r, --remove= - the list of group or users to remove from the permission +# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. +# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO +# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. +# +# Requires YunoHost version 3.7.0 or higher. +ynh_permission_update() { + # Declare an array to define the options of this helper. + local legacy_args=parltP + local -A args_array=([p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected=) + local permission + local add + local remove + local label + local show_tile + local protected + ynh_handle_getopts_args "$@" + add=${add:-} + remove=${remove:-} + label=${label:-} + show_tile=${show_tile:-} + protected=${protected:-} + + if [[ -n $add ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --add alice bob + # will be: + # add=['alice', 'bob'] + add=",add=['${add//';'/"','"}']" + fi + if [[ -n $remove ]]; then + # Convert a list from getopts to python list + # Note that getopts separate the args with ';' + # For example: + # --remove alice bob + # will be: + # remove=['alice', 'bob'] + remove=",remove=['${remove//';'/"','"}']" + fi + + if [[ -n $label ]]; then + label=",label='$label'" + fi + + if [[ -n $show_tile ]]; then + if [ $show_tile == "true" ]; then + show_tile=",show_tile=True" + else + show_tile=",show_tile=False" + fi + fi + + if [[ -n $protected ]]; then + if [ $protected == "true" ]; then + protected=",protected=True" + else + protected=",protected=False" + fi + fi + + yunohost tools shell -c "from yunohost.permission import user_permission_update; user_permission_update('$app.$permission' $add $remove $label $show_tile $protected , force=True)" +} + +# Check if a permission has an user +# +# example: ynh_permission_has_user --permission=main --user=visitors +# +# usage: ynh_permission_has_user --permission=permission --user=user +# | arg: -p, --permission= - the permission to check +# | arg: -u, --user= - the user seek in the permission +# | exit: Return 1 if the permission doesn't have that user or doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.7.1 or higher. +ynh_permission_has_user() { + local legacy_args=pu + # Declare an array to define the options of this helper. + local -A args_array=([p]=permission= [u]=user=) + local permission + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! ynh_permission_exists --permission=$permission; then + return 1 + fi + + # Check both allowed and corresponding_users sections in the json + for section in "allowed" "corresponding_users"; do + if yunohost user permission info "$app.$permission" --output-as json --quiet \ + | jq -e --arg user $user --arg section $section '.[$section] | index($user)' >/dev/null; then + return 0 + fi + done + + return 1 +} + +# Check if a legacy permissions exist +# +# [packagingv1] +# +# usage: ynh_legacy_permissions_exists +# | exit: Return 1 if the permission doesn't exist, 0 otherwise +# +# Requires YunoHost version 4.1.2 or higher. +ynh_legacy_permissions_exists() { + for permission in "skipped" "unprotected" "protected"; do + if ynh_permission_exists --permission="legacy_${permission}_uris"; then + return 0 + fi + done + return 1 +} + +# Remove all legacy permissions +# +# [packagingv1] +# +# usage: ynh_legacy_permissions_delete_all +# +# example: +# if ynh_legacy_permissions_exists +# then +# ynh_legacy_permissions_delete_all +# # You can recreate the required permissions here with ynh_permission_create +# fi +# Requires YunoHost version 4.1.2 or higher. +ynh_legacy_permissions_delete_all() { + for permission in "skipped" "unprotected" "protected"; do + if ynh_permission_exists --permission="legacy_${permission}_uris"; then + ynh_permission_delete --permission="legacy_${permission}_uris" + fi + done +} diff --git a/helpers/helpers.v2.1.d/php b/helpers/helpers.v2.1.d/php new file mode 100644 index 000000000..7fbe3f1ba --- /dev/null +++ b/helpers/helpers.v2.1.d/php @@ -0,0 +1,583 @@ +#!/bin/bash + +readonly YNH_DEFAULT_PHP_VERSION=7.4 +# Declare the actual PHP version to use. +# A packager willing to use another version of PHP can override the variable into its _common.sh. +YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} + +# Create a dedicated PHP-FPM config +# +# usage: ynh_add_fpm_config +# +# Case 1 (recommended) : your provided a snippet conf/extra_php-fpm.conf +# +# The actual PHP configuration will be automatically generated, +# and your extra_php-fpm.conf will be appended (typically contains PHP upload limits) +# +# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf +# +# Performance-related options in the PHP conf, such as : +# pm.max_children, pm.start_servers, pm.min_spare_servers pm.max_spare_servers +# are computed from two parameters called "usage" and "footprint" which can be set to low/medium/high. (cf details below) +# +# If you wish to tweak those, please initialize the settings `fpm_usage` and `fpm_footprint` +# *prior* to calling this helper. Otherwise, "low" will be used as a default for both values. +# +# Otherwise, if you want the user to have control over these, we encourage to create a config panel +# (which should ultimately be standardized by the core ...) +# +# Case 2 (deprecate) : you provided an entire conf/php-fpm.conf +# +# The configuration will be hydrated, replacing __FOOBAR__ placeholders with $foobar values, etc. +# +# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf +# +# ---------------------- +# +# fpm_footprint: Memory footprint of the service (low/medium/high). +# low - Less than 20 MB of RAM by pool. +# medium - Between 20 MB and 40 MB of RAM by pool. +# high - More than 40 MB of RAM by pool. +# N - Or you can specify a quantitative footprint as MB by pool (use watch -n0.5 ps -o user,cmd,%cpu,rss -u APP) +# +# fpm_usage: Expected usage of the service (low/medium/high). +# low - Personal usage, behind the SSO. +# medium - Low usage, few people or/and publicly accessible. +# high - High usage, frequently visited website. +# +# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM. +# So it will be used to defined 'pm.max_children' +# A lower value for the footprint will allow more children for 'pm.max_children'. And so for +# 'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the +# value of 'pm.max_children' +# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores. +# +# The usage value will defined the way php will handle the children for the pool. +# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the +# service is used, otherwise no child will stay alive. This config gives the lower footprint when the +# service is idle. But will use more proc since it has to start a child as soon it's used. +# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children +# equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request. +# The number of children can grow if needed. The footprint can stay low if the service is idle, but +# not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few +# children already available. +# Set as 'high', the process manager will be set at 'static'. There will be always as many children as +# 'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum +# RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many +# children ready to answer. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_fpm_config() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vufpdg + local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service [g]=group=) + local group + local phpversion + local usage + local footprint + local package + local dedicated_service + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package=${package:-} + group=${group:-} + + # The default behaviour is to use the template. + local autogenconf=false + usage="${usage:-}" + footprint="${footprint:-}" + if [ -n "$usage" ] || [ -n "$footprint" ] || [[ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]]; then + autogenconf=true + + # If no usage provided, default to the value existing in setting ... or to low + local fpm_usage_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_usage) + if [ -z "$usage" ] + then + usage=${fpm_usage_in_setting:-low} + ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage + fi + + # If no footprint provided, default to the value existing in setting ... or to low + local fpm_footprint_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_footprint) + if [ -z "$footprint" ] + then + footprint=${fpm_footprint_in_setting:-low} + ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint + fi + + fi + # Do not use a dedicated service by default + dedicated_service=${dedicated_service:-0} + + # Set the default PHP-FPM version by default + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + + # If the PHP version changed, remove the old fpm conf + # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper) + if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then + local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf" + + if [[ -f "$old_php_finalphpconf" ]] + then + ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf" + ynh_remove_fpm_config + fi + fi + + # Legacy args (packager should just list their php dependency as regular apt dependencies... + if [ -n "$package" ]; then + # Install the additionnal packages from the default repository + ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future" + ynh_install_app_dependencies "$package" + fi + + if [ $dedicated_service -eq 1 ]; then + ynh_print_warn --message "Argument --dedicated_service of ynh_add_fpm_config is deprecated and to be removed in the future" + local fpm_service="${app}-phpfpm" + local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" + else + local fpm_service="php${phpversion}-fpm" + local fpm_config_dir="/etc/php/$phpversion/fpm" + fi + + # Create the directory for FPM pools + mkdir --parents "$fpm_config_dir/pool.d" + + ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" + ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" + ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service" + ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion + + # Migrate from mutual PHP service to dedicated one. + if [ $dedicated_service -eq 1 ]; then + local old_fpm_config_dir="/etc/php/$phpversion/fpm" + # If a config file exist in the common pool, move it. + if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]; then + ynh_print_info --message="Migrate to a dedicated php-fpm service for $app." + # Create a backup of the old file before migration + ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf" + # Remove the old PHP config file + ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf" + # Reload PHP to release the socket and allow the dedicated service to use it + ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload + fi + fi + + if [ $autogenconf == "false" ]; then + # Usage 1, use the template in conf/php-fpm.conf + local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" + # Make sure now that the template indeed exists + [ -e "$phpfpm_path" ] || ynh_die --message="Unable to find template to configure PHP-FPM." + else + # Usage 2, generate a PHP-FPM config file with ynh_get_scalable_phpfpm + + # Define the values to use for the configuration of PHP. + ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint + + local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app") + local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" + echo " +[__APP__] + +user = __APP__ +group = __PHPFPM_GROUP__ + +chdir = __INSTALL_DIR__ + +listen = /var/run/php/php__PHPVERSION__-fpm-__APP__.sock +listen.owner = www-data +listen.group = www-data + +pm = __PHP_PM__ +pm.max_children = __PHP_MAX_CHILDREN__ +pm.max_requests = 500 +request_terminate_timeout = 1d +" >"$phpfpm_path" + + if [ "$php_pm" = "dynamic" ]; then + echo " +pm.start_servers = __PHP_START_SERVERS__ +pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__ +pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__ +" >>"$phpfpm_path" + + elif [ "$php_pm" = "ondemand" ]; then + echo " +pm.process_idle_timeout = 10s +" >>"$phpfpm_path" + fi + + # Concatene the extra config. + if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then + cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path" + fi + fi + + local finalphpconf="$fpm_config_dir/pool.d/$app.conf" + ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" + + if [ -e "$YNH_APP_BASEDIR/conf/php-fpm.ini" ]; then + ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." + ynh_add_config --template="php-fpm.ini" --destination="$fpm_config_dir/conf.d/20-$app.ini" + fi + + if [ $dedicated_service -eq 1 ]; then + # Create a dedicated php-fpm.conf for the service + local globalphpconf=$fpm_config_dir/php-fpm-$app.conf + + echo "[global] +pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid +error_log = /var/log/php/fpm-php.__APP__.log +syslog.ident = php-fpm-__APP__ +include = __FINALPHPCONF__ +" >$YNH_APP_BASEDIR/conf/php-fpm-$app.conf + + ynh_add_config --template="php-fpm-$app.conf" --destination="$globalphpconf" + + # Create a config for a dedicated PHP-FPM service for the app + echo "[Unit] +Description=PHP __PHPVERSION__ FastCGI Process Manager for __APP__ +After=network.target + +[Service] +Type=notify +PIDFile=/run/php/php__PHPVERSION__-fpm-__APP__.pid +ExecStart=/usr/sbin/php-fpm__PHPVERSION__ --nodaemonize --fpm-config __GLOBALPHPCONF__ +ExecReload=/bin/kill -USR2 \$MAINPID + +[Install] +WantedBy=multi-user.target +" >$YNH_APP_BASEDIR/conf/$fpm_service + + # Create this dedicated PHP-FPM service + ynh_add_systemd_config --service=$fpm_service --template=$fpm_service + # Integrate the service in YunoHost admin panel + yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --description "Php-fpm dedicated to $app" + # Configure log rotate + ynh_use_logrotate --logfile=/var/log/php + # Restart the service, as this service is either stopped or only for this app + ynh_systemd_action --service_name=$fpm_service --action=restart + else + # Validate that the new php conf doesn't break php-fpm entirely + if ! php-fpm${phpversion} --test 2>/dev/null; then + php-fpm${phpversion} --test || true + ynh_secure_remove --file="$finalphpconf" + ynh_die --message="The new configuration broke php-fpm?" + fi + ynh_systemd_action --service_name=$fpm_service --action=reload + fi +} + +# Remove the dedicated PHP-FPM config +# +# usage: ynh_remove_fpm_config +# +# Requires YunoHost version 2.7.2 or higher. +ynh_remove_fpm_config() { + local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) + local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) + dedicated_service=${dedicated_service:-0} + # Get the version of PHP used by this app + local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) + + # Assume default PHP-FPM version by default + phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}" + + # Assume default PHP files if not set + if [ -z "$fpm_config_dir" ]; then + fpm_config_dir="/etc/php/$YNH_DEFAULT_PHP_VERSION/fpm" + fpm_service="php$YNH_DEFAULT_PHP_VERSION-fpm" + fi + + ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" + if [ -e $fpm_config_dir/conf.d/20-$app.ini ]; then + ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" + fi + + if [ $dedicated_service -eq 1 ]; then + # Remove the dedicated service PHP-FPM service for the app + ynh_remove_systemd_config --service=$fpm_service + # Remove the global PHP-FPM conf + ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" + # Remove the service from the list of services known by YunoHost + yunohost service remove $fpm_service + elif ynh_package_is_installed --package="php${phpversion}-fpm"; then + ynh_systemd_action --service_name=$fpm_service --action=reload + fi + + # If the PHP version used is not the default version for YunoHost + # The second part with YNH_APP_PURGE is an ugly hack to guess that we're inside the remove script + # (we don't actually care about its value, we just check its not empty hence it exists) + if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + # Remove app dependencies ... but ideally should happen via an explicit call from packager + ynh_remove_app_dependencies + fi +} + +# Install another version of PHP. +# +# [internal] +# +# Legacy, to be remove on bullseye +# +# usage: ynh_install_php --phpversion=phpversion [--package=packages] +# | arg: -v, --phpversion= - Version of PHP to install. +# | arg: -p, --package= - Additionnal PHP packages to install +# +# Requires YunoHost version 3.8.1 or higher. +ynh_install_php() { + # Declare an array to define the options of this helper. + local legacy_args=vp + local -A args_array=([v]=phpversion= [p]=package=) + local phpversion + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package=${package:-} + + if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]; then + ynh_die --message="Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION" + fi + + ynh_install_app_dependencies "$package" +} + +# Remove the specific version of PHP used by the app. +# +# [internal] +# +# Legacy, to be remove on bullseye +# +# usage: ynh_remove_php +# +# Requires YunoHost version 3.8.1 or higher. +ynh_remove_php () { + ynh_remove_app_dependencies +} + +# Define the values to configure PHP-FPM +# +# [internal] +# +# usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print] +# | arg: -f, --footprint= - Memory footprint of the service (low/medium/high). +# low - Less than 20 MB of RAM by pool. +# medium - Between 20 MB and 40 MB of RAM by pool. +# high - More than 40 MB of RAM by pool. +# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value. +# To have this value, use the following command and stress the service. +# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP +# +# | arg: -u, --usage= - Expected usage of the service (low/medium/high). +# low - Personal usage, behind the SSO. +# medium - Low usage, few people or/and publicly accessible. +# high - High usage, frequently visited website. +# +# | arg: -p, --print - Print the result (intended for debug purpose only when packaging the app) +ynh_get_scalable_phpfpm() { + local legacy_args=ufp + # Declare an array to define the options of this helper. + local -A args_array=([u]=usage= [f]=footprint= [p]=print) + local usage + local footprint + local print + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Set all characters as lowercase + footprint=${footprint,,} + usage=${usage,,} + print=${print:-0} + + if [ "$footprint" = "low" ]; then + footprint=20 + elif [ "$footprint" = "medium" ]; then + footprint=35 + elif [ "$footprint" = "high" ]; then + footprint=50 + fi + + # Define the factor to determine min_spare_servers + # to avoid having too few children ready to start for heavy apps + if [ $footprint -le 20 ]; then + min_spare_servers_factor=8 + elif [ $footprint -le 35 ]; then + min_spare_servers_factor=5 + else + min_spare_servers_factor=3 + fi + + # Define the way the process manager handle child processes. + if [ "$usage" = "low" ]; then + php_pm=ondemand + elif [ "$usage" = "medium" ]; then + php_pm=dynamic + elif [ "$usage" = "high" ]; then + php_pm=static + else + ynh_die --message="Does not recognize '$usage' as an usage value." + fi + + # Get the total of RAM available, except swap. + local max_ram=$(ynh_get_ram --total --ignore_swap) + + at_least_one() { + # Do not allow value below 1 + if [ $1 -le 0 ]; then + echo 1 + else + echo $1 + fi + } + + # Define pm.max_children + # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. + # So if PHP-FPM start the maximum of children, it won't exceed half of the ram. + php_max_children=$(($max_ram / 2 / $footprint)) + # If process manager is set as static, use half less children. + # Used as static, there's always as many children as the value of pm.max_children + if [ "$php_pm" = "static" ]; then + php_max_children=$(($php_max_children / 2)) + fi + php_max_children=$(at_least_one $php_max_children) + + # To not overload the proc, limit the number of children to 4 times the number of cores. + local core_number=$(nproc) + local max_proc=$(($core_number * 4)) + if [ $php_max_children -gt $max_proc ]; then + php_max_children=$max_proc + fi + + # Get a potential forced value for php_max_children + local php_forced_max_children=$(ynh_app_setting_get --app=$app --key=php_forced_max_children) + if [ -n "$php_forced_max_children" ]; then + php_max_children=$php_forced_max_children + fi + + if [ "$php_pm" = "dynamic" ]; then + # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager + php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor)) + php_min_spare_servers=$(at_least_one $php_min_spare_servers) + + php_max_spare_servers=$(($php_max_children / 2)) + php_max_spare_servers=$(at_least_one $php_max_spare_servers) + + php_start_servers=$(($php_min_spare_servers + ($php_max_spare_servers - $php_min_spare_servers) / 2)) + php_start_servers=$(at_least_one $php_start_servers) + else + php_min_spare_servers=0 + php_max_spare_servers=0 + php_start_servers=0 + fi + + if [ $print -eq 1 ]; then + ynh_print_warn --message="Footprint=${footprint}Mb by pool." + ynh_print_warn --message="Process manager=$php_pm" + ynh_print_warn --message="Max RAM=${max_ram}Mb" + if [ "$php_pm" != "static" ]; then + ynh_print_warn --message="\nMax estimated footprint=$(($php_max_children * $footprint))" + ynh_print_warn --message="Min estimated footprint=$(($php_min_spare_servers * $footprint))" + fi + if [ "$php_pm" = "dynamic" ]; then + ynh_print_warn --message="Estimated average footprint=$(($php_max_spare_servers * $footprint))" + elif [ "$php_pm" = "static" ]; then + ynh_print_warn --message="Estimated footprint=$(($php_max_children * $footprint))" + fi + ynh_print_warn --message="\nRaw php-fpm values:" + ynh_print_warn --message="pm.max_children = $php_max_children" + if [ "$php_pm" = "dynamic" ]; then + ynh_print_warn --message="pm.start_servers = $php_start_servers" + ynh_print_warn --message="pm.min_spare_servers = $php_min_spare_servers" + ynh_print_warn --message="pm.max_spare_servers = $php_max_spare_servers" + fi + fi +} + +readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17 +# Declare the actual composer version to use. +# A packager willing to use another version of composer can override the variable into its _common.sh. +YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} + +# Execute a command with Composer +# +# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$install_dir] --commands="commands" +# | arg: -v, --phpversion - PHP version to use with composer +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path +# | arg: -c, --commands - Commands to execute. +# +# Requires YunoHost version 4.2 or higher. +ynh_composer_exec() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vwc + declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=) + local phpversion + local workdir + local commands + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + workdir="${workdir:-${install_dir:-$final_path}}" + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ + php${phpversion} "$workdir/composer.phar" $commands \ + -d "$workdir" --no-interaction --no-ansi 2>&1 +} + +# Install and initialize Composer in the given directory +# +# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$install_dir] [--install_args="--optimize-autoloader"] [--composerversion=composerversion] +# | arg: -v, --phpversion - PHP version to use with composer +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir. +# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include +# | arg: -c, --composerversion - Composer version to install +# +# Requires YunoHost version 4.2 or higher. +ynh_install_composer() { + local _globalphpversion=${phpversion-:} + # Declare an array to define the options of this helper. + local legacy_args=vwac + declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=) + local phpversion + local workdir + local install_args + local composerversion + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + workdir="${workdir:-$final_path}" + else + workdir="${workdir:-$install_dir}" + fi + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + phpversion="${phpversion:-$YNH_PHP_VERSION}" + else + phpversion="${phpversion:-$_globalphpversion}" + fi + + install_args="${install_args:-}" + composerversion="${composerversion:-$YNH_COMPOSER_VERSION}" + + curl -sS https://getcomposer.org/installer \ + | COMPOSER_HOME="$workdir/.composer" \ + php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \ + || ynh_die --message="Unable to install Composer." + + # install dependencies + ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \ + || ynh_die --message="Unable to install core dependencies with Composer." +} diff --git a/helpers/helpers.v2.1.d/postgresql b/helpers/helpers.v2.1.d/postgresql new file mode 100644 index 000000000..35b95cd5f --- /dev/null +++ b/helpers/helpers.v2.1.d/postgresql @@ -0,0 +1,310 @@ +#!/bin/bash + +PSQL_ROOT_PWD_FILE=/etc/yunohost/psql +PSQL_VERSION=13 + +# Open a connection as a user +# +# usage: ynh_psql_connect_as --user=user --password=password [--database=database] +# | arg: -u, --user= - the user name to connect as +# | arg: -p, --password= - the user password +# | arg: -d, --database= - the database to connect to +# +# examples: +# ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" +# ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_connect_as() { + # Declare an array to define the options of this helper. + local legacy_args=upd + local -A args_array=([u]=user= [p]=password= [d]=database=) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" +} + +# Execute a command as root user +# +# usage: ynh_psql_execute_as_root --sql=sql [--database=database] +# | arg: -s, --sql= - the SQL command to execute +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_execute_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=sd + local -A args_array=([s]=sql= [d]=database=) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ + $database <<<"$sql" +} + +# Execute a command from a file as root user +# +# usage: ynh_psql_execute_file_as_root --file=file [--database=database] +# | arg: -f, --file= - the file containing SQL commands +# | arg: -d, --database= - the database to connect to +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_execute_file_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=fd + local -A args_array=([f]=file= [d]=database=) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + if [ -n "$database" ]; then + database="--database=$database" + fi + + ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \ + $database <"$file" +} + +# Create a database and grant optionnaly privilegies to a user +# +# [internal] +# +# usage: ynh_psql_create_db db [user] +# | arg: db - the database name to create +# | arg: user - the user to grant privilegies +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_create_db() { + local db=$1 + local user=${2:-} + + local sql="CREATE DATABASE ${db};" + + # grant all privilegies to user + if [ -n "$user" ]; then + sql+="ALTER DATABASE ${db} OWNER TO ${user};" + sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" + fi + + ynh_psql_execute_as_root --sql="$sql" +} + +# Drop a database +# +# [internal] +# +# If you intend to drop the database *and* the associated user, +# consider using ynh_psql_remove_db instead. +# +# usage: ynh_psql_drop_db db +# | arg: db - the database name to drop +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_drop_db() { + local db=$1 + # First, force disconnection of all clients connected to the database + # https://stackoverflow.com/questions/17449420/postgresql-unable-to-drop-database-because-of-some-auto-connections-to-db + ynh_psql_execute_as_root --sql="REVOKE CONNECT ON DATABASE $db FROM public;" --database="$db" + ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db' AND pid <> pg_backend_pid();" --database="$db" + sudo --login --user=postgres dropdb $db +} + +# Dump a database +# +# usage: ynh_psql_dump_db --database=database +# | arg: -d, --database= - the database name to dump +# | ret: the psqldump output +# +# example: ynh_psql_dump_db 'roundcube' > ./dump.sql +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo --login --user=postgres pg_dump "$database" +} + +# Create a user +# +# [internal] +# +# usage: ynh_psql_create_user user pwd +# | arg: user - the user name to create +# | arg: pwd - the password to identify user by +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_create_user() { + local user=$1 + local pwd=$2 + ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" +} + +# Check if a psql user exists +# +# [packagingv1] +# +# usage: ynh_psql_user_exists --user=user +# | arg: -u, --user= - the user for which to check existence +# | exit: Return 1 if the user doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=user=) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user"; then + return 1 + else + return 0 + fi +} + +# Check if a psql database exists +# +# usage: ynh_psql_database_exists --database=database +# | arg: -d, --database= - the database for which to check existence +# | exit: Return 1 if the database doesn't exist, 0 otherwise +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_database_exists() { + # Declare an array to define the options of this helper. + local legacy_args=d + local -A args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # if psql is not there, we cannot check the db + # though it could exists. + if ! command -v psql + then + ynh_print_err -m "PostgreSQL is not installed, impossible to check for db existence." + return 1 + elif ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then + return 1 + else + return 0 + fi +} + +# Drop a user +# +# [internal] +# +# usage: ynh_psql_drop_user user +# | arg: user - the user name to drop +# +# Requires YunoHost version 3.5.0 or higher. +ynh_psql_drop_user() { + ynh_psql_execute_as_root --sql="DROP USER ${1};" +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# [packagingv1] +# +# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# | arg: -p, --db_pwd= - Password of the database. If not provided, a password will be generated +# +# After executing this helper, the password of the created database will be available in $db_pwd +# It will also be stored as "psqlpwd" into the app settings. +# +# Requires YunoHost version 2.7.13 or higher. +ynh_psql_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! ynh_psql_user_exists --user=$db_user; then + local new_db_pwd=$(ynh_string_random) # Generate a random password + # If $db_pwd is not provided, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + ynh_psql_create_user "$db_user" "$db_pwd" + elif [ -z $db_pwd ]; then + ynh_die --message="The user $db_user exists, please provide his password" + fi + + ynh_psql_create_db "$db_name" "$db_user" # Create the database + ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config +} + +# Remove a database if it exists, and the associated user +# +# [packagingv1] +# +# usage: ynh_psql_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user= - Owner of the database +# | arg: -n, --db_name= - Name of the database +# +# Requires YunoHost version 2.7.13 or higher. +ynh_psql_remove_db() { + # Declare an array to define the options of this helper. + local legacy_args=un + local -A args_array=([u]=db_user= [n]=db_name=) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ynh_psql_database_exists --database=$db_name; then # Check if the database exists + ynh_psql_drop_db $db_name # Remove the database + else + ynh_print_warn --message="Database $db_name not found" + fi + + # Remove psql user if it exists + if ynh_psql_user_exists --user=$db_user; then + ynh_psql_drop_user $db_user + else + ynh_print_warn --message="User $db_user not found" + fi +} + +# Create a master password and set up global settings +# +# [internal] +# +# usage: ynh_psql_test_if_first_run +# +# It also make sure that postgresql is installed and running +# Please always call this script in install and restore scripts +# +# Requires YunoHost version 2.7.13 or higher. +ynh_psql_test_if_first_run() { + + # Make sure postgresql is indeed installed + dpkg --list | grep -q "ii postgresql-$PSQL_VERSION" || ynh_die --message="postgresql-$PSQL_VERSION is not installed !?" + + yunohost tools regen-conf postgresql +} diff --git a/helpers/helpers.v2.1.d/redis b/helpers/helpers.v2.1.d/redis new file mode 100644 index 000000000..545bb8705 --- /dev/null +++ b/helpers/helpers.v2.1.d/redis @@ -0,0 +1,39 @@ +#!/bin/bash + +# get the first available redis database +# +# usage: ynh_redis_get_free_db +# | returns: the database number to use +ynh_redis_get_free_db() { + local result max db + result=$(redis-cli INFO keyspace) + + # get the num + max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+") + + db=0 + # default Debian setting is 15 databases + for i in $(seq 0 "$max") + do + if ! echo "$result" | grep -q "db$i" + then + db=$i + break 1 + fi + db=-1 + done + + test "$db" -eq -1 && ynh_die --message="No available Redis databases..." + + echo "$db" +} + +# Create a master password and set up global settings +# Please always call this script in install and restore scripts +# +# usage: ynh_redis_remove_db database +# | arg: database - the database to erase +ynh_redis_remove_db() { + local db=$1 + redis-cli -n "$db" flushdb +} diff --git a/helpers/helpers.v2.1.d/ruby b/helpers/helpers.v2.1.d/ruby new file mode 100644 index 000000000..82a946935 --- /dev/null +++ b/helpers/helpers.v2.1.d/ruby @@ -0,0 +1,306 @@ +#!/bin/bash + +rbenv_install_dir="/opt/rbenv" +ruby_version_path="$rbenv_install_dir/versions" + +# RBENV_ROOT is the directory of rbenv, it needs to be loaded as a environment variable. +export RBENV_ROOT="$rbenv_install_dir" +export rbenv_root="$rbenv_install_dir" + +if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then + build_ruby_dependencies="libjemalloc-dev curl build-essential libreadline-dev zlib1g-dev libsqlite3-dev libssl-dev libxml2-dev libxslt-dev autoconf automake bison libtool" + build_pkg_dependencies="${build_pkg_dependencies:-} $build_ruby_dependencies" +fi + +# Load the version of Ruby for an app, and set variables. +# +# ynh_use_ruby has to be used in any app scripts before using Ruby for the first time. +# This helper will provide alias and variables to use in your scripts. +# +# To use gem or Ruby, use the alias `ynh_gem` and `ynh_ruby` +# Those alias will use the correct version installed for the app +# For example: use `ynh_gem install` instead of `gem install` +# +# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_ruby` +# And propagate $PATH to sudo with $ynh_ruby_load_path +# Exemple: `ynh_exec_as $app $ynh_ruby_load_path $ynh_gem install` +# +# $PATH contains the path of the requested version of Ruby. +# However, $PATH is duplicated into $ruby_path to outlast any manipulation of $PATH +# You can use the variable `$ynh_ruby_load_path` to quickly load your Ruby version +# in $PATH for an usage into a separate script. +# Exemple: $ynh_ruby_load_path $final_path/script_that_use_gem.sh` +# +# +# Finally, to start a Ruby service with the correct version, 2 solutions +# Either the app is dependent of Ruby or gem, but does not called it directly. +# In such situation, you need to load PATH +# `Environment="__YNH_RUBY_LOAD_PATH__"` +# `ExecStart=__FINALPATH__/my_app` +# You will replace __YNH_RUBY_LOAD_PATH__ with $ynh_ruby_load_path +# +# Or Ruby start the app directly, then you don't need to load the PATH variable +# `ExecStart=__YNH_RUBY__ my_app run` +# You will replace __YNH_RUBY__ with $ynh_ruby +# +# +# one other variable is also available +# - $ruby_path: The absolute path to Ruby binaries for the chosen version. +# +# usage: ynh_use_ruby +# +# Requires YunoHost version 3.2.2 or higher. +ynh_use_ruby () { + ruby_version=$(ynh_app_setting_get --app=$app --key=ruby_version) + + # Get the absolute path of this version of Ruby + ruby_path="$ruby_version_path/$YNH_APP_INSTANCE_NAME/bin" + + # Allow alias to be used into bash script + shopt -s expand_aliases + + # Create an alias for the specific version of Ruby and a variable as fallback + ynh_ruby="$ruby_path/ruby" + alias ynh_ruby="$ynh_ruby" + # And gem + ynh_gem="$ruby_path/gem" + alias ynh_gem="$ynh_gem" + + # Load the path of this version of Ruby in $PATH + if [[ :$PATH: != *":$ruby_path"* ]]; then + PATH="$ruby_path:$PATH" + fi + # Create an alias to easily load the PATH + ynh_ruby_load_path="PATH=$PATH" + + # Sets the local application-specific Ruby version + pushd ${install_dir:-$final_path} + $rbenv_install_dir/bin/rbenv local $ruby_version + popd +} + +# Install a specific version of Ruby +# +# ynh_install_ruby will install the version of Ruby provided as argument by using rbenv. +# +# This helper creates a /etc/profile.d/rbenv.sh that configures PATH environment for rbenv +# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) +# +# Don't forget to execute ruby-dependent command in a login environment +# (e.g. sudo --login option) +# When not possible (e.g. in systemd service definition), please use direct path +# to rbenv shims (e.g. $RBENV_ROOT/shims/bundle) +# +# usage: ynh_install_ruby --ruby_version=ruby_version +# | arg: -v, --ruby_version= - Version of ruby to install. +# +# Requires YunoHost version 3.2.2 or higher. +ynh_install_ruby () { + # Declare an array to define the options of this helper. + local legacy_args=v + local -A args_array=( [v]=ruby_version= ) + local ruby_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Load rbenv path in PATH + local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Ruby prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Move an existing Ruby binary, to avoid to block rbenv + test -x /usr/bin/ruby && mv /usr/bin/ruby /usr/bin/ruby_rbenv + + # Install or update rbenv + mkdir -p $rbenv_install_dir + rbenv="$(command -v rbenv $rbenv_install_dir/bin/rbenv | grep "$rbenv_install_dir/bin/rbenv" | head -1)" + if [ -n "$rbenv" ]; then + pushd "${rbenv%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/rbenv/rbenv.git"; then + ynh_print_info --message="Updating rbenv..." + git pull -q --tags origin master + ynh_ruby_try_bash_extension + else + ynh_print_info --message="Reinstalling rbenv..." + cd .. + ynh_secure_remove --file=$rbenv_install_dir + mkdir -p $rbenv_install_dir + cd $rbenv_install_dir + git init -q + git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 + git checkout -q -b master origin/master + ynh_ruby_try_bash_extension + rbenv=$rbenv_install_dir/bin/rbenv + fi + popd + else + ynh_print_info --message="Installing rbenv..." + pushd $rbenv_install_dir + git init -q + git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1 + git checkout -q -b master origin/master + ynh_ruby_try_bash_extension + rbenv=$rbenv_install_dir/bin/rbenv + popd + fi + + mkdir -p "${rbenv_install_dir}/plugins" + + ruby_build="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-install rbenv-install | head -1)" + if [ -n "$ruby_build" ]; then + pushd "${ruby_build%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/rbenv/ruby-build.git"; then + ynh_print_info --message="Updating ruby-build..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing ruby-build..." + git clone -q https://github.com/rbenv/ruby-build.git "${rbenv_install_dir}/plugins/ruby-build" + fi + + rbenv_alias="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)" + if [ -n "$rbenv_alias" ]; then + pushd "${rbenv_alias%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/tpope/rbenv-aliases.git"; then + ynh_print_info --message="Updating rbenv-aliases..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing rbenv-aliases..." + git clone -q https://github.com/tpope/rbenv-aliases.git "${rbenv_install_dir}/plugins/rbenv-aliase" + fi + + rbenv_latest="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)" + if [ -n "$rbenv_latest" ]; then + pushd "${rbenv_latest%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then + ynh_print_info --message="Updating xxenv-latest..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing xxenv-latest..." + git clone -q https://github.com/momo-lab/xxenv-latest.git "${rbenv_install_dir}/plugins/xxenv-latest" + fi + + # Enable caching + mkdir -p "${rbenv_install_dir}/cache" + + # Create shims directory if needed + mkdir -p "${rbenv_install_dir}/shims" + + # Restore /usr/local/bin in PATH + PATH=$CLEAR_PATH + + # And replace the old Ruby binary + test -x /usr/bin/ruby_rbenv && mv /usr/bin/ruby_rbenv /usr/bin/ruby + + # Install the requested version of Ruby + local final_ruby_version=$(rbenv latest --print $ruby_version) + if ! [ -n "$final_ruby_version" ]; then + final_ruby_version=$ruby_version + fi + ynh_print_info --message="Installing Ruby $final_ruby_version" + RUBY_CONFIGURE_OPTS="--disable-install-doc --with-jemalloc" MAKE_OPTS="-j2" rbenv install --skip-existing $final_ruby_version > /dev/null 2>&1 + + # Store ruby_version into the config of this app + ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=ruby_version --value=$final_ruby_version + + # Remove app virtualenv + if rbenv alias --list | grep --quiet "$YNH_APP_INSTANCE_NAME " + then + rbenv alias $YNH_APP_INSTANCE_NAME --remove + fi + + # Create app virtualenv + rbenv alias $YNH_APP_INSTANCE_NAME $final_ruby_version + + # Cleanup Ruby versions + ynh_cleanup_ruby + + # Set environment for Ruby users + echo "#rbenv +export RBENV_ROOT=$rbenv_install_dir +export PATH=\"$rbenv_install_dir/bin:$PATH\" +eval \"\$(rbenv init -)\" +#rbenv" > /etc/profile.d/rbenv.sh + + # Load the environment + eval "$(rbenv init -)" +} + +# Remove the version of Ruby used by the app. +# +# This helper will also cleanup Ruby versions +# +# usage: ynh_remove_ruby +ynh_remove_ruby () { + local ruby_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=ruby_version) + + # Load rbenv path in PATH + local CLEAR_PATH="$rbenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Ruby prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + rbenv alias $YNH_APP_INSTANCE_NAME --remove + + # Remove the line for this app + ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=ruby_version + + # Cleanup Ruby versions + ynh_cleanup_ruby +} + +# Remove no more needed versions of Ruby used by the app. +# +# This helper will check what Ruby version are no more required, +# and uninstall them +# If no app uses Ruby, rbenv will be also removed. +# +# usage: ynh_cleanup_ruby +ynh_cleanup_ruby () { + + # List required Ruby versions + local installed_apps=$(yunohost app list | grep -oP 'id: \K.*$') + local required_ruby_versions="" + for installed_app in $installed_apps + do + local installed_app_ruby_version=$(ynh_app_setting_get --app=$installed_app --key="ruby_version") + if [[ -n "$installed_app_ruby_version" ]] + then + required_ruby_versions="${installed_app_ruby_version}\n${required_ruby_versions}" + fi + done + + # Remove no more needed Ruby versions + local installed_ruby_versions=$(rbenv versions --bare --skip-aliases | grep -Ev '/') + for installed_ruby_version in $installed_ruby_versions + do + if ! echo ${required_ruby_versions} | grep -q "${installed_ruby_version}" + then + ynh_print_info --message="Removing Ruby-$installed_ruby_version" + $rbenv_install_dir/bin/rbenv uninstall --force $installed_ruby_version + fi + done + + # If none Ruby version is required + if [[ -z "$required_ruby_versions" ]] + then + # Remove rbenv environment configuration + ynh_print_info --message="Removing rbenv" + ynh_secure_remove --file="$rbenv_install_dir" + ynh_secure_remove --file="/etc/profile.d/rbenv.sh" + fi +} + +ynh_ruby_try_bash_extension() { + if [ -x src/configure ]; then + src/configure && make -C src || { + ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." + } + fi +} diff --git a/helpers/helpers.v2.1.d/setting b/helpers/helpers.v2.1.d/setting new file mode 100644 index 000000000..82a5d274e --- /dev/null +++ b/helpers/helpers.v2.1.d/setting @@ -0,0 +1,160 @@ +#!/bin/bash + +# Get an application setting +# +# usage: ynh_app_setting_get --app=app --key=key +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting to get +# +# Requires YunoHost version 2.2.4 or higher. +ynh_app_setting_get() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=ak + local -A args_array=([a]=app= [k]=key=) + local app + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [[ $key =~ (unprotected|protected|skipped)_ ]]; then + yunohost app setting $app $key + else + ynh_app_setting "get" "$app" "$key" + fi +} + +# Set an application setting +# +# usage: ynh_app_setting_set --app=app --key=key --value=value +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting name to set +# | arg: -v, --value= - the setting value to set +# +# Requires YunoHost version 2.2.4 or higher. +ynh_app_setting_set() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=akv + local -A args_array=([a]=app= [k]=key= [v]=value=) + local app + local key + local value + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [[ $key =~ (unprotected|protected|skipped)_ ]]; then + yunohost app setting $app $key -v $value + else + ynh_app_setting "set" "$app" "$key" "$value" + fi +} + +# Delete an application setting +# +# usage: ynh_app_setting_delete --app=app --key=key +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting to delete +# +# Requires YunoHost version 2.2.4 or higher. +ynh_app_setting_delete() { + local _globalapp=${app-:} + # Declare an array to define the options of this helper. + local legacy_args=ak + local -A args_array=([a]=app= [k]=key=) + local app + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + app="${app:-$_globalapp}" + + if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then + yunohost app setting $app $key -d + else + ynh_app_setting "delete" "$app" "$key" + fi +} + +# Small "hard-coded" interface to avoid calling "yunohost app" directly each +# time dealing with a setting is needed (which may be so slow on ARM boards) +# +# [internal] +# +ynh_app_setting() { + set +o xtrace # set +x + ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - </dev/null \ + | tr --complement --delete "$filter" \ + | sed --quiet 's/\(.\{'"$length"'\}\).*/\1/p' +} + +# Substitute/replace a string (or expression) by another in a file +# +# usage: ynh_replace_string --match_string=match_string --replace_string=replace_string --target_file=target_file +# | arg: -m, --match_string= - String to be searched and replaced in the file +# | arg: -r, --replace_string= - String that will replace matches +# | arg: -f, --target_file= - File in which the string will be replaced. +# +# As this helper is based on sed command, regular expressions and references to +# sub-expressions can be used (see sed manual page for more information) +# +# Requires YunoHost version 2.6.4 or higher. +ynh_replace_string() { + # Declare an array to define the options of this helper. + local legacy_args=mrf + local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) + local match_string + local replace_string + local target_file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + set +o xtrace # set +x + + local delimit=$'\001' + # Escape the delimiter if it's in the string. + match_string=${match_string//${delimit}/"\\${delimit}"} + replace_string=${replace_string//${delimit}/"\\${delimit}"} + + set -o xtrace # set -x + sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file" +} + +# Substitute/replace a special string by another in a file +# +# usage: ynh_replace_special_string --match_string=match_string --replace_string=replace_string --target_file=target_file +# | arg: -m, --match_string= - String to be searched and replaced in the file +# | arg: -r, --replace_string= - String that will replace matches +# | arg: -t, --target_file= - File in which the string will be replaced. +# +# This helper will use ynh_replace_string, but as you can use special +# characters, you can't use some regular expressions and sub-expressions. +# +# Requires YunoHost version 2.7.7 or higher. +ynh_replace_special_string() { + # Declare an array to define the options of this helper. + local legacy_args=mrf + local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=) + local match_string + local replace_string + local target_file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Escape any backslash to preserve them as simple backslash. + match_string=${match_string//\\/"\\\\"} + replace_string=${replace_string//\\/"\\\\"} + + # Escape the & character, who has a special function in sed. + match_string=${match_string//&/"\&"} + replace_string=${replace_string//&/"\&"} + + ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file" +} + +# Sanitize a string intended to be the name of a database +# +# [packagingv1] +# +# usage: ynh_sanitize_dbid --db_name=name +# | arg: -n, --db_name= - name to correct/sanitize +# | ret: the corrected name +# +# example: dbname=$(ynh_sanitize_dbid $app) +# +# Underscorify the string (replace - and . by _) +# +# Requires YunoHost version 2.2.4 or higher. +ynh_sanitize_dbid() { + # Declare an array to define the options of this helper. + local legacy_args=n + local -A args_array=([n]=db_name=) + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # We should avoid having - and . in the name of databases. They are replaced by _ + echo ${db_name//[-.]/_} +} + +# Normalize the url path syntax +# +# [internal] +# +# Handle the slash at the beginning of path and its absence at ending +# Return a normalized url path +# +# examples: +# url_path=$(ynh_normalize_url_path $url_path) +# ynh_normalize_url_path example # -> /example +# ynh_normalize_url_path /example # -> /example +# ynh_normalize_url_path /example/ # -> /example +# ynh_normalize_url_path / # -> / +# +# usage: ynh_normalize_url_path --path_url=path_to_normalize +# | arg: -p, --path_url= - URL path to normalize before using it +# +# Requires YunoHost version 2.6.4 or higher. +ynh_normalize_url_path() { + # Declare an array to define the options of this helper. + local legacy_args=p + local -A args_array=([p]=path_url=) + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." + if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / + path_url="/$path_url" # Add / at begin of path variable + fi + if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then # If the last character is a / and that not the only character. + path_url="${path_url:0:${#path_url}-1}" # Delete the last character + fi + echo $path_url +} diff --git a/helpers/helpers.v2.1.d/systemd b/helpers/helpers.v2.1.d/systemd new file mode 100644 index 000000000..765c575ef --- /dev/null +++ b/helpers/helpers.v2.1.d/systemd @@ -0,0 +1,189 @@ +#!/bin/bash + +# Create a dedicated systemd config +# +# usage: ynh_add_systemd_config [--service=service] [--template=template] +# | arg: -s, --service= - Service name (optionnal, `$app` by default) +# | arg: -t, --template= - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template) +# +# This will use the template `../conf/.service`. +# +# See the documentation of `ynh_add_config` for a description of the template +# format and how placeholders are replaced with actual variables. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_systemd_config() { + # Declare an array to define the options of this helper. + local legacy_args=stv + local -A args_array=([s]=service= [t]=template=) + local service + local template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + service="${service:-$app}" + template="${template:-systemd.service}" + + ynh_add_config --template="$template" --destination="/etc/systemd/system/$service.service" + + systemctl enable $service --quiet + systemctl daemon-reload +} + +# Remove the dedicated systemd config +# +# usage: ynh_remove_systemd_config [--service=service] +# | arg: -s, --service= - Service name (optionnal, $app by default) +# +# Requires YunoHost version 2.7.2 or higher. +ynh_remove_systemd_config() { + # Declare an array to define the options of this helper. + local legacy_args=s + local -A args_array=([s]=service=) + local service + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local service="${service:-$app}" + + local finalsystemdconf="/etc/systemd/system/$service.service" + if [ -e "$finalsystemdconf" ]; then + ynh_systemd_action --service_name=$service --action=stop + systemctl disable $service --quiet + ynh_secure_remove --file="$finalsystemdconf" + systemctl daemon-reload + fi +} + +# Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started +# +# usage: ynh_systemd_action [--service_name=service_name] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] +# | arg: -n, --service_name= - Name of the service to start. Default : `$app` +# | arg: -a, --action= - Action to perform with systemctl. Default: start +# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. +# | arg: -p, --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` +# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. +# | arg: -e, --length= - Length of the error log displayed for debugging : Default : 20 +# +# Requires YunoHost version 3.5.0 or higher. +ynh_systemd_action() { + # Declare an array to define the options of this helper. + local legacy_args=nalpte + local -A args_array=([n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=) + local service_name + local action + local line_match + local length + local log_path + local timeout + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + service_name="${service_name:-$app}" + action=${action:-start} + line_match=${line_match:-} + length=${length:-20} + log_path="${log_path:-/var/log/$service_name/$service_name.log}" + timeout=${timeout:-300} + + # Manage case of service already stopped + if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service_name; then + return 0 + fi + + # Start to read the log + if [[ -n "$line_match" ]]; then + local templog="$(mktemp)" + # Following the starting of the app in its log + if [ "$log_path" == "systemd" ]; then + # Read the systemd journal + journalctl --unit=$service_name --follow --since=-0 --quiet >"$templog" & + # Get the PID of the journalctl command + local pid_tail=$! + else + # Read the specified log file + tail --follow=name --retry --lines=0 "$log_path" >"$templog" 2>&1 & + # Get the PID of the tail command + local pid_tail=$! + fi + fi + + # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. + if [ "$action" == "reload" ]; then + action="reload-or-restart" + fi + + local time_start="$(date --utc --rfc-3339=seconds | cut -d+ -f1) UTC" + + # If the service fails to perform the action + if ! systemctl $action $service_name; then + # Show syslog for this service + ynh_exec_err journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name + # If a log is specified for this service, show also the content of this log + if [ -e "$log_path" ]; then + ynh_exec_err tail --lines=$length "$log_path" + fi + ynh_clean_check_starting + return 1 + fi + + # Start the timeout and try to find line_match + if [[ -n "${line_match:-}" ]]; then + set +x + local i=0 + local starttime=$(date +%s) + for i in $(seq 1 $timeout); do + # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout + if [ "$log_path" == "systemd" ]; then + # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action + if journalctl --unit=$service_name --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then + ynh_print_info --message="The service $service_name has correctly executed the action ${action}." + break + fi + else + if grep --extended-regexp --quiet "$line_match" "$templog"; then + ynh_print_info --message="The service $service_name has correctly executed the action ${action}." + break + fi + fi + if [ $i -eq 30 ]; then + echo "(this may take some time)" >&2 + fi + # Also check the timeout using actual timestamp, because sometimes for some reason, + # journalctl may take a huge time to run, and we end up waiting literally an entire hour + # instead of 5 min ... + if [[ "$(( $(date +%s) - $starttime))" -gt "$timeout" ]] + then + i=$timeout + break + fi + sleep 1 + done + set -x + if [ $i -ge 3 ]; then + echo "" >&2 + fi + if [ $i -eq $timeout ]; then + ynh_print_warn --message="The service $service_name didn't fully executed the action ${action} before the timeout." + ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" + ynh_exec_warn journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name + if [ -e "$log_path" ]; then + ynh_print_warn --message="\-\-\-" + ynh_exec_warn tail --lines=$length "$log_path" + fi + fi + ynh_clean_check_starting + fi +} + +# Clean temporary process and file used by ynh_check_starting +# +# [internal] +# +# Requires YunoHost version 3.5.0 or higher. +ynh_clean_check_starting() { + if [ -n "${pid_tail:-}" ]; then + # Stop the execution of tail. + kill -SIGTERM $pid_tail 2>&1 + fi + if [ -n "${templog:-}" ]; then + ynh_secure_remove --file="$templog" 2>&1 + fi +} diff --git a/helpers/helpers.v2.1.d/user b/helpers/helpers.v2.1.d/user new file mode 100644 index 000000000..e608a3308 --- /dev/null +++ b/helpers/helpers.v2.1.d/user @@ -0,0 +1,196 @@ +#!/bin/bash + +# Check if a YunoHost user exists +# +# usage: ynh_user_exists --username=username +# | arg: -u, --username= - the username to check +# | ret: 0 if the user exists, 1 otherwise. +# +# example: ynh_user_exists 'toto' || echo "User does not exist" +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null +} + +# Retrieve a YunoHost user information +# +# usage: ynh_user_get_info --username=username --key=key +# | arg: -u, --username= - the username to retrieve info from +# | arg: -k, --key= - the key to retrieve +# | ret: the value associate to that key +# +# example: mail=$(ynh_user_get_info --username="toto" --key=mail) +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_get_info() { + # Declare an array to define the options of this helper. + local legacy_args=uk + local -A args_array=([u]=username= [k]=key=) + local username + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user info "$username" --output-as json --quiet | jq -r ".$key" +} + +# Get the list of YunoHost users +# +# usage: ynh_user_list +# | ret: one username per line as strings +# +# example: for u in $(ynh_user_list); do ... ; done +# +# Requires YunoHost version 2.4.0 or higher. +ynh_user_list() { + yunohost user list --output-as json --quiet | jq -r ".users | keys[]" +} + +# Check if a user exists on the system +# +# [packagingv1] +# +# usage: ynh_system_user_exists --username=username +# | arg: -u, --username= - the username to check +# | ret: 0 if the user exists, 1 otherwise. +# +# Requires YunoHost version 2.2.4 or higher. +ynh_system_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent passwd "$username" &>/dev/null +} + +# Check if a group exists on the system +# +# [packagingv1] +# +# usage: ynh_system_group_exists --group=group +# | arg: -g, --group= - the group to check +# | ret: 0 if the group exists, 1 otherwise. +# +# Requires YunoHost version 3.5.0.2 or higher. +ynh_system_group_exists() { + # Declare an array to define the options of this helper. + local legacy_args=g + local -A args_array=([g]=group=) + local group + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent group "$group" &>/dev/null +} + +# Create a system user +# +# [packagingv1] +# +# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell] [--groups="group1 group2"] +# | arg: -u, --username= - Name of the system user that will be create +# | arg: -h, --home_dir= - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home +# | arg: -s, --use_shell - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell +# | arg: -g, --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp) +# +# Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability) : +# ``` +# ynh_system_user_create --username=nextcloud +# ``` +# Create a discourse user using /var/www/discourse as home directory and the default login shell : +# ``` +# ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell +# ``` +# +# Requires YunoHost version 2.6.4 or higher. +ynh_system_user_create() { + # Declare an array to define the options of this helper. + local legacy_args=uhs + local -A args_array=([u]=username= [h]=home_dir= [s]=use_shell [g]=groups=) + local username + local home_dir + local use_shell + local groups + + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + use_shell="${use_shell:-0}" + home_dir="${home_dir:-}" + groups="${groups:-}" + + if ! ynh_system_user_exists "$username"; then # Check if the user exists on the system + # If the user doesn't exist + if [ -n "$home_dir" ]; then # If a home dir is mentioned + local user_home_dir="--home-dir $home_dir" + else + local user_home_dir="--no-create-home" + fi + if [ $use_shell -eq 1 ]; then # If we want a shell for the user + local shell="" # Use default shell + else + local shell="--shell /usr/sbin/nologin" + fi + useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account" + fi + + local group + for group in $groups; do + usermod -a -G "$group" "$username" + done +} + +# Delete a system user +# +# [packagingv1] +# +# usage: ynh_system_user_delete --username=user_name +# | arg: -u, --username= - Name of the system user that will be create +# +# Requires YunoHost version 2.6.4 or higher. +ynh_system_user_delete() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Check if the user exists on the system + if ynh_system_user_exists "$username"; then + deluser $username + else + ynh_print_warn --message="The user $username was not found" + fi + + # Check if the group exists on the system + if ynh_system_group_exists "$username"; then + delgroup $username + fi +} + +# Execute a command as another user +# +# usage: ynh_exec_as $USER COMMAND [ARG ...] +# +# Requires YunoHost version 4.1.7 or higher. +ynh_exec_as() { + local user=$1 + shift 1 + + if [[ $user = $(whoami) ]]; then + eval "$@" + else + sudo -u "$user" "$@" + fi +} diff --git a/helpers/helpers.v2.1.d/utils b/helpers/helpers.v2.1.d/utils new file mode 100644 index 000000000..631e154e2 --- /dev/null +++ b/helpers/helpers.v2.1.d/utils @@ -0,0 +1,1104 @@ +#!/bin/bash + +YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)} + +# Handle script crashes / failures +# +# [internal] +# +# usage: +# ynh_exit_properly is used only by the helper ynh_abort_if_errors. +# You should not use it directly. +# Instead, add to your script: +# ynh_clean_setup () { +# instructions... +# } +# +# This function provide a way to clean some residual of installation that not managed by remove script. +# +# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script +# +# Requires YunoHost version 2.6.4 or higher. +ynh_exit_properly() { + local exit_code=$? + + if [[ "${YNH_APP_ACTION:-}" =~ ^install$|^upgrade$|^restore$ ]] + then + rm -rf "/var/cache/yunohost/download/" + fi + + if [ "$exit_code" -eq 0 ]; then + exit 0 # Exit without error if the script ended correctly + fi + + trap '' EXIT # Ignore new exit signals + # Do not exit anymore if a command fail or if a variable is empty + set +o errexit # set +e + set +o nounset # set +u + + # Small tempo to avoid the next message being mixed up with other DEBUG messages + sleep 0.5 + + if type -t ynh_clean_setup >/dev/null; then # Check if the function exist in the app script. + ynh_clean_setup # Call the function to do specific cleaning for the app. + fi + + # Exit with error status + # We don't call ynh_die basically to avoid unecessary 10-ish + # debug lines about parsing args and stuff just to exit 1.. + exit 1 +} + +# Exits if an error occurs during the execution of the script. +# +# [packagingv1] +# +# usage: ynh_abort_if_errors +# +# This configure the rest of the script execution such that, if an error occurs +# or if an empty variable is used, the execution of the script stops immediately +# and a call to `ynh_clean_setup` is triggered if it has been defined by your script. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_abort_if_errors() { + set -o errexit # set -e; Exit if a command fail + set -o nounset # set -u; And if a variable is used unset + trap ynh_exit_properly EXIT # Capturing exit signals on shell script +} + +# When running an app script with packaging format >= 2, auto-enable ynh_abort_if_errors except for remove script +if [[ "${YNH_CONTEXT:-}" != "regenconf" ]] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 && [[ ${YNH_APP_ACTION} != "remove" ]] +then + ynh_abort_if_errors +fi + +# Download, check integrity, uncompress and patch the source from app.src +# +# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] +# | arg: -d, --dest_dir= - Directory where to setup sources +# | arg: -s, --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise +# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders) +# | arg: -r, --full_replace= - Remove previous sources before installing new sources (can be 1 or 0, default to 0) +# +# #### New 'sources' resources +# +# (See also the resources documentation which may be more complete?) +# +# This helper will read infos from the 'sources' resources in the manifest.toml of the app +# and expect a structure like: +# +# ```toml +# [resources.sources] +# [resources.sources.main] +# url = "https://some.address.to/download/the/app/archive" +# sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL +# ``` +# +# ##### Optional flags +# +# ```text +# format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract +# "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract +# "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract +# "whatever" # an arbitrary value, not really meaningful except to imply that the file won't be extracted +# +# in_subdir = true # default, there's an intermediate subdir in the archive before accessing the actual files +# false # sources are directly in the archive root +# n # (special cases) an integer representing a number of subdirs levels to get rid of +# +# extract = true # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ... +# = false # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset. +# # in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value +# +# rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical +# platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for +# ``` +# +# You may also define assets url and checksum per-architectures such as: +# ```toml +# [resources.sources] +# [resources.sources.main] +# amd64.url = "https://some.address.to/download/the/app/archive/when/amd64" +# amd64.sha256 = "0123456789abcdef" +# armhf.url = "https://some.address.to/download/the/app/archive/when/armhf" +# armhf.sha256 = "fedcba9876543210" +# ``` +# +# In which case ynh_setup_source --dest_dir="$install_dir" will automatically pick the appropriate source depending on the arch +# +# +# +# #### Legacy format '.src' +# +# This helper will read `conf/${source_id}.src`, download and install the sources. +# +# The src file need to contains: +# ``` +# SOURCE_URL=Address to download the app archive +# SOURCE_SUM=Sha256 sum +# SOURCE_FORMAT=tar.gz +# SOURCE_IN_SUBDIR=false +# SOURCE_FILENAME=example.tar.gz +# SOURCE_EXTRACT=(true|false) +# SOURCE_PLATFORM=linux/arm64/v8 +# ``` +# +# The helper will: +# - Download the specific URL if there is no local archive +# - Check the integrity with the specific sha256 sum +# - Uncompress the archive to `$dest_dir`. +# - If `in_subdir` is true, the first level directory of the archive will be removed. +# - If `in_subdir` is a numeric value, the N first level directories will be removed. +# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir` +# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir +# +# Requires YunoHost version 2.6.4 or higher. +ynh_setup_source() { + # Declare an array to define the options of this helper. + local legacy_args=dsk + local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) + local dest_dir + local source_id + local keep + local full_replace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + keep="${keep:-}" + full_replace="${full_replace:-0}" + + if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null + then + source_id="${source_id:-main}" + local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") + if jq -re ".url" <<< "$sources_json" + then + local arch_prefix="" + else + local arch_prefix=".$YNH_ARCH" + fi + + local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" + local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" + local src_sumprg="sha256sum" + local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" + local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" + local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" + local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" + local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" + + [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" + [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" + + if [[ -z "$src_format" ]] + then + if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]] + then + src_format="zip" + elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]] + then + src_format="tar.gz" + elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]] + then + src_format="tar.xz" + elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]] + then + src_format="tar.bz2" + elif [[ -z "$src_extract" ]] + then + src_extract="false" + fi + fi + else + source_id="${source_id:-app}" + local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src" + + # Load value from configuration file (see above for a small doc about this file + # format) + local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_rename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-) + local src_platform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-) + fi + + # Default value + src_sumprg=${src_sumprg:-sha256sum} + src_in_subdir=${src_in_subdir:-true} + src_format=${src_format:-tar.gz} + src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]') + src_extract=${src_extract:-true} + + if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]] + then + ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter" + fi + + + # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... + local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" + + # Gotta use this trick with 'dirname' because source_id may contain slashes x_x + mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) + src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" + + if [ "$src_format" = "docker" ]; then + src_platform="${src_platform:-"linux/$YNH_ARCH"}" + else + if test -e "$local_src"; then + cp $local_src $src_filename + fi + + [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" + + # If the file was prefetched but somehow doesn't match the sum, rm and redownload it + if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + rm -f "$src_filename" + fi + + # Only redownload the file if it wasnt prefetched + if [ ! -e "$src_filename" ] + then + # NB. we have to declare the var as local first, + # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work + # because local always return 0 ... + local out + # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget) + out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \ + || ynh_die --message="$out" + fi + + # Check the control sum + if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status + then + local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_size="$(du -hs ${src_filename} | cut --fields=1)" + rm -f ${src_filename} + ynh_die --message="Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})." + fi + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ + if [ -n "$keep" ] && [ -e "$dest_dir" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + mkdir -p $keep_dir + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$dest_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")" + cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep" + fi + done + fi + + if [ "$full_replace" -eq 1 ]; then + ynh_secure_remove --file="$dest_dir" + fi + + # Extract source into the app dir + mkdir --parents "$dest_dir" + + if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then + _ynh_apply_default_permissions $dest_dir + fi + if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then + _ynh_apply_default_permissions $dest_dir + fi + + if [[ "$src_extract" == "false" ]]; then + if [[ -z "$src_rename" ]] + then + mv $src_filename $dest_dir + else + mv $src_filename $dest_dir/$src_rename + fi + elif [[ "$src_format" == "docker" ]]; then + "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1 + elif [[ "$src_format" == "zip" ]]; then + # Zip format + # Using of a temp directory, because unzip doesn't manage --strip-components + if $src_in_subdir; then + local tmp_dir=$(mktemp --directory) + unzip -quo $src_filename -d "$tmp_dir" + cp --archive $tmp_dir/*/. "$dest_dir" + ynh_secure_remove --file="$tmp_dir" + else + unzip -quo $src_filename -d "$dest_dir" + fi + ynh_secure_remove --file="$src_filename" + else + local strip="" + if [ "$src_in_subdir" != "false" ]; then + if [ "$src_in_subdir" == "true" ]; then + local sub_dirs=1 + else + local sub_dirs="$src_in_subdir" + fi + strip="--strip-components $sub_dirs" + fi + if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then + tar --extract --file=$src_filename --directory="$dest_dir" $strip + else + ynh_die --message="Archive format unrecognized." + fi + ynh_secure_remove --file="$src_filename" + fi + + # Apply patches + if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then + local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/) + if (($(find $patches_folder -type f -name "${source_id}-*.patch" 2>/dev/null | wc --lines) > "0")); then + pushd "$dest_dir" + for p in $patches_folder/${source_id}-*.patch; do + echo $p + patch --strip=1 <$p || ynh_print_warn --message="Packagers /!\\ patch $p failed to apply" + done + popd + fi + fi + + # Add supplementary files + if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then + cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir" + fi + + # Keep files to be backup/restored at the end of the helper + # Assuming $dest_dir already exists + if [ -n "$keep" ]; then + local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID} + local stuff_to_keep + for stuff_to_keep in $keep; do + if [ -e "$keep_dir/$stuff_to_keep" ]; then + mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")" + + # We add "--no-target-directory" (short option is -T) to handle the special case + # when we "keep" a folder, but then the new setup already contains the same dir (but possibly empty) + # in which case a regular "cp" will create a copy of the directory inside the directory ... + # resulting in something like /var/www/$app/data/data instead of /var/www/$app/data + # cf https://unix.stackexchange.com/q/94831 for a more elaborate explanation on the option + cp --archive --no-target-directory "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep" + fi + done + fi + rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/ +} + +# Curl abstraction to help with POST requests to local pages (such as installation forms) +# +# usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ... +# | arg: page_uri - Path (relative to `$path_url`) of the page where POST data will be sent +# | arg: key1=value1 - (Optionnal) POST key and corresponding value +# | arg: key2=value2 - (Optionnal) Another POST key and corresponding value +# | arg: ... - (Optionnal) More POST keys and values +# +# example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2" +# +# For multiple calls, cookies are persisted between each call for the same app +# +# `$domain` and `$path_url` should be defined externally (and correspond to the domain.tld and the /path (of the app?)) +# +# Requires YunoHost version 2.6.4 or higher. +ynh_local_curl() { + # Define url of page to curl + local local_page=$(ynh_normalize_url_path $1) + local full_path=$path_url$local_page + + if [ "${path_url}" == "/" ]; then + full_path=$local_page + fi + + local full_page_url=https://localhost$full_path + + # Concatenate all other arguments with '&' to prepare POST data + local POST_data="" + local arg="" + for arg in "${@:2}"; do + POST_data="${POST_data}${arg}&" + done + if [ -n "$POST_data" ]; then + # Add --data arg and remove the last character, which is an unecessary '&' + POST_data="--data ${POST_data::-1}" + fi + + # Wait untils nginx has fully reloaded (avoid curl fail with http2) + sleep 2 + + local cookiefile=/tmp/ynh-$app-cookie.txt + touch $cookiefile + chown root $cookiefile + chmod 700 $cookiefile + + # Temporarily enable visitors if needed... + local visitors_enabled=$(ynh_permission_has_user "main" "visitors" && echo yes || echo no) + if [[ $visitors_enabled == "no" ]]; then + ynh_permission_update --permission "main" --add "visitors" + fi + + # Curl the URL + curl --silent --show-error --insecure --location --header "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" --cookie-jar $cookiefile --cookie $cookiefile + + if [[ $visitors_enabled == "no" ]]; then + ynh_permission_update --permission "main" --remove "visitors" + fi +} + +# Create a dedicated config file from a template +# +# usage: ynh_add_config --template="template" --destination="destination" +# | arg: -t, --template= - Template config file to use +# | arg: -d, --destination= - Destination of the config file +# +# examples: +# ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" +# ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" +# +# The template can be by default the name of a file in the conf directory +# of a YunoHost Package, a relative path or an absolute path. +# +# The helper will use the template `template` to generate a config file +# `destination` by replacing the following keywords with global variables +# that should be defined before calling this helper : +# ``` +# __PATH__ by $path_url +# __NAME__ by $app +# __NAMETOCHANGE__ by $app +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# ``` +# And any dynamic variables that should be defined before calling this helper like: +# ``` +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# ``` +# +# The helper will verify the checksum and backup the destination file +# if it's different before applying the new template. +# +# And it will calculate and store the destination file checksum +# into the app settings when configuration is done. +# +# Requires YunoHost version 4.1.0 or higher. +ynh_add_config() { + # Declare an array to define the options of this helper. + local legacy_args=tdv + local -A args_array=([t]=template= [d]=destination=) + local template + local destination + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local template_path + + if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then + template_path="$YNH_APP_BASEDIR/conf/$template" + elif [ -f "$template" ]; then + template_path=$template + else + ynh_die --message="The provided template $template doesn't exist" + fi + + ynh_backup_if_checksum_is_different --file="$destination" + + # Make sure to set the permissions before we copy the file + # This is to cover a case where an attacker could have + # created a file beforehand to have control over it + # (cp won't overwrite ownership / modes by default...) + touch $destination + chown root:root $destination + chmod 640 $destination + + cp -f "$template_path" "$destination" + + _ynh_apply_default_permissions $destination + + ynh_replace_vars --file="$destination" + + ynh_store_file_checksum --file="$destination" +} + +# Replace variables in a file +# +# [internal] +# +# usage: ynh_replace_vars --file="file" +# | arg: -f, --file= - File where to replace variables +# +# The helper will replace the following keywords with global variables +# that should be defined before calling this helper : +# __PATH__ by $path_url +# __NAME__ by $app +# __NAMETOCHANGE__ by $app +# __USER__ by $app +# __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource) +# __YNH_NODE_LOAD_PATH__ by $ynh_node_load_PATH +# +# And any dynamic variables that should be defined before calling this helper like: +# __DOMAIN__ by $domain +# __APP__ by $app +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# +# Requires YunoHost version 4.1.0 or higher. +ynh_replace_vars() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Replace specific YunoHost variables + if test -n "${path_url:-}"; then + # path_url_slash_less is path_url, or a blank value if path_url is only '/' + local path_url_slash_less=${path_url%/} + ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file" + ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file" + fi + if test -n "${app:-}"; then + ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$file" + ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file" + ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" + fi + # Legacy + if test -n "${final_path:-}"; then + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file" + ynh_replace_string --match_string="__INSTALL_DIR__" --replace_string="$final_path" --target_file="$file" + fi + # Legacy / Packaging v1 only + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2 && test -n "${YNH_PHP_VERSION:-}"; then + ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file" + fi + if test -n "${ynh_node_load_PATH:-}"; then + ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file" + fi + + # Replace others variables + + # List other unique (__ __) variables in $file + local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g")) + + set +o xtrace # set +x + + # Do the replacement + local delimit=@ + for one_var in "${uniques_vars[@]}"; do + # Validate that one_var is indeed defined + # -v checks if the variable is defined, for example: + # -v FOO tests if $FOO is defined + # -v $FOO tests if ${!FOO} is defined + # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 + [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" + + # Escape delimiter in match/replace string + match_string="__${one_var^^}__" + match_string=${match_string//${delimit}/"\\${delimit}"} + replace_string="${!one_var}" + replace_string=${replace_string//\\/\\\\} + replace_string=${replace_string//${delimit}/"\\${delimit}"} + + # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs) + sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file" + done + set -o xtrace # set -x +} + +# Get a value from heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_read_var_in_file --file=PATH --key=KEY +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to get +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# This helpers match several var affectation use case in several languages +# We don't use jq or equivalent to keep comments and blank space in files +# This helpers work line by line, it is not able to work correctly +# if you have several identical keys in your files +# +# Example of line this helpers can managed correctly +# .yml +# title: YunoHost documentation +# email: 'yunohost@yunohost.org' +# .json +# "theme": "colib'ris", +# "port": 8102 +# "some_boolean": false, +# "user": null +# .ini +# some_boolean = On +# action = "Clear" +# port = 20 +# .php +# $user= +# user => 20 +# .py +# USER = 8102 +# user = 'https://donate.local' +# CUSTOM['user'] = 'YunoHost' +# +# Requires YunoHost version 4.3 or higher. +ynh_read_var_in_file() { + # Declare an array to define the options of this helper. + local legacy_args=fka + local -A args_array=([f]=file= [k]=key= [a]=after=) + local file + local key + local after + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + after="${after:-}" + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local line_number=1 + if [[ -n "$after" ]]; then + line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + echo YNH_NULL + return 0 + fi + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + + local first_char="${expression:0:1}" + if [[ "$first_char" == '"' ]]; then + echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g' + elif [[ "$first_char" == "'" ]]; then + echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g" + else + echo "$expression" + fi + set -o xtrace # set -x +} + +# Set a value into heterogeneous file (yaml, json, php, python...) +# +# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE +# | arg: -f, --file= - the path to the file +# | arg: -k, --key= - the key to set +# | arg: -v, --value= - the value to set +# | arg: -a, --after= - the line just before the key (in case of multiple lines with the name of the key in the file) +# +# Requires YunoHost version 4.3 or higher. +ynh_write_var_in_file() { + # Declare an array to define the options of this helper. + local legacy_args=fkva + local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=) + local file + local key + local value + local after + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + after="${after:-}" + + [[ -f $file ]] || ynh_die --message="File $file does not exists" + + set +o xtrace # set +x + + # Get the line number after which we search for the variable + local after_line_number=1 + if [[ -n "$after" ]]; then + after_line_number=$(grep -m1 -n $after $file | cut -d: -f1) + if [[ -z "$after_line_number" ]]; then + set -o xtrace # set -x + return 1 + fi + fi + + local filename="$(basename -- "$file")" + local ext="${filename##*.}" + local endline=',;' + local assign="=>|:|=" + local comments="#" + local string="\"'" + if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then + endline='#' + fi + if [[ "$ext" =~ ^ini|env$ ]]; then + comments="[;#]" + fi + if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then + comments="//" + fi + local list='\[\s*['$string']?\w+['$string']?\]' + local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*' + var_part+="[$string]?${key}[$string]?" + var_part+='\s*\]?\s*' + var_part+="($assign)" + var_part+='\s*' + + # Extract the part after assignation sign + local expression_with_comment="$((tail +$after_line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)" + if [[ "$expression_with_comment" == "YNH_NULL" ]]; then + set -o xtrace # set -x + return 1 + fi + local value_line_number="$(tail +$after_line_number ${file} | grep -m1 -n -i -P $var_part'\K.*$' | cut -d: -f1)" + value_line_number=$((after_line_number + value_line_number)) + local range="${after_line_number},${value_line_number} " + + # Remove comments if needed + local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")" + endline=${expression_with_comment#"$expression"} + endline="$(echo "$endline" | sed 's/\\/\\\\/g')" + value="$(echo "$value" | sed 's/\\/\\\\/g')" + value=${value//&/"\&"} + local first_char="${expression:0:1}" + delimiter=$'\001' + if [[ "$first_char" == '"' ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # So we need \\\\ to go through 2 sed + value="$(echo "$value" | sed 's/"/\\\\"/g')" + sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file} + elif [[ "$first_char" == "'" ]]; then + # \ and sed is quite complex you need 2 \\ to get one in a sed + # However double quotes implies to double \\ to + # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str + value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")" + sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file} + else + if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]]; then + value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"' + fi + if [[ "$ext" =~ ^yaml|yml$ ]]; then + value=" $value" + fi + sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file} + fi + set -o xtrace # set -x +} + +# Render templates with Jinja2 +# +# [internal] +# +# Attention : Variables should be exported before calling this helper to be +# accessible inside templates. +# +# usage: ynh_render_template some_template output_path +# | arg: some_template - Template file to be rendered +# | arg: output_path - The path where the output will be redirected to +ynh_render_template() { + local template_path=$1 + local output_path=$2 + mkdir -p "$(dirname $output_path)" + # Taken from https://stackoverflow.com/a/35009576 + python3 -c 'import os, sys, jinja2; sys.stdout.write( + jinja2.Template(sys.stdin.read() + ).render(os.environ));' <$template_path >$output_path +} + +# Fetch the Debian release codename +# +# [packagingv1] +# +# usage: ynh_get_debian_release +# | ret: The Debian release codename (i.e. jessie, stretch, ...) +# +# Requires YunoHost version 2.7.12 or higher. +ynh_get_debian_release() { + echo $(lsb_release --codename --short) +} + +_acceptable_path_to_delete() { + local file=$1 + + local forbidden_paths=$(ls -d / /* /{var,home,usr}/* /etc/{default,sudoers.d,yunohost,cron*} /etc/yunohost/{apps,domains,hooks.d} /opt/yunohost 2> /dev/null) + + # Legacy : A couple apps still have data in /home/$app ... + if [[ -n "${app:-}" ]] + then + forbidden_paths=$(echo "$forbidden_paths" | grep -v "/home/$app") + fi + + # Use realpath to normalize the path .. + # i.e convert ///foo//bar//..///baz//// to /foo/baz + file=$(realpath --no-symlinks "$file") + if [ -z "$file" ] || grep -q -x -F "$file" <<< "$forbidden_paths"; then + return 1 + else + return 0 + fi +} + +# Remove a file or a directory securely +# +# usage: ynh_secure_remove --file=path_to_remove +# | arg: -f, --file= - File or directory to remove +# +# Requires YunoHost version 2.6.4 or higher. +ynh_secure_remove() { + # Declare an array to define the options of this helper. + local legacy_args=f + local -A args_array=([f]=file=) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + set +o xtrace # set +x + + if [ $# -ge 2 ]; then + ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." + fi + + if [[ -z "$file" ]]; then + ynh_print_warn --message="ynh_secure_remove called with empty argument, ignoring." + elif [[ ! -e $file ]]; then + ynh_print_info --message="'$file' wasn't deleted because it doesn't exist." + elif ! _acceptable_path_to_delete "$file"; then + ynh_print_warn --message="Not deleting '$file' because it is not an acceptable path to delete." + else + rm --recursive "$file" + fi + + set -o xtrace # set -x +} + +# Read the value of a key in a ynh manifest file +# +# usage: ynh_read_manifest --manifest="manifest.json" --key="key" +# | arg: -m, --manifest= - Path of the manifest to read +# | arg: -k, --key= - Name of the key to find +# | ret: the value associate to that key +# +# Requires YunoHost version 3.5.0 or higher. +ynh_read_manifest() { + # Declare an array to define the options of this helper. + local legacy_args=mk + local -A args_array=([m]=manifest= [k]=manifest_key=) + local manifest + local manifest_key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [ ! -e "${manifest:-}" ]; then + # If the manifest isn't found, try the common place for backup and restore script. + if [ -e "$YNH_APP_BASEDIR/manifest.json" ] + then + manifest="$YNH_APP_BASEDIR/manifest.json" + elif [ -e "$YNH_APP_BASEDIR/manifest.toml" ] + then + manifest="$YNH_APP_BASEDIR/manifest.toml" + else + ynh_die --message "No manifest found !?" + fi + fi + + if echo "$manifest" | grep -q '\.json$' + then + jq ".$manifest_key" "$manifest" --raw-output + else + cat "$manifest" | python3 -c 'import json, toml, sys; print(json.dumps(toml.load(sys.stdin)))' | jq ".$manifest_key" --raw-output + fi +} + +# Read the upstream version from the manifest or `$YNH_APP_MANIFEST_VERSION` +# +# usage: ynh_app_upstream_version [--manifest="manifest.json"] +# | arg: -m, --manifest= - Path of the manifest to read +# | ret: the version number of the upstream app +# +# If the `manifest` is not specified, the envvar `$YNH_APP_MANIFEST_VERSION` will be used. +# +# The version number in the manifest is defined by `~ynh`. +# +# For example, if the manifest contains `4.3-2~ynh3` the function will return `4.3-2` +# +# Requires YunoHost version 3.5.0 or higher. +ynh_app_upstream_version() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=manifest=) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + manifest="${manifest:-}" + + if [[ "$manifest" != "" ]] && [[ -e "$manifest" ]]; then + version_key_=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + else + version_key_=$YNH_APP_MANIFEST_VERSION + fi + + echo "${version_key_/~ynh*/}" +} + +# Read package version from the manifest +# +# [internal] +# +# usage: ynh_app_package_version [--manifest="manifest.json"] +# | arg: -m, --manifest= - Path of the manifest to read +# | ret: the version number of the package +# +# The version number in the manifest is defined by `~ynh`. +# +# For example, if the manifest contains `4.3-2~ynh3` the function will return `3` +# +# Requires YunoHost version 3.5.0 or higher. +ynh_app_package_version() { + # Declare an array to define the options of this helper. + local legacy_args=m + local -A args_array=([m]=manifest=) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + version_key_=$YNH_APP_MANIFEST_VERSION + echo "${version_key_/*~ynh/}" +} + +# Checks the app version to upgrade with the existing app version and returns: +# +# usage: ynh_check_app_version_changed +# | ret: `UPGRADE_APP` if the upstream version changed, `UPGRADE_PACKAGE` otherwise. +# +# This helper should be used to avoid an upgrade of an app, or the upstream part +# of it, when it's not needed +# +# Requires YunoHost version 3.5.0 or higher. +ynh_check_app_version_changed() { + local return_value=${YNH_APP_UPGRADE_TYPE} + + if [ "$return_value" == "UPGRADE_SAME" ] || [ "$return_value" == "DOWNGRADE" ]; then + return_value="UPGRADE_APP" + fi + + echo $return_value +} + +# Compare the current package version against another version given as an argument. +# +# usage: ynh_compare_current_package_version --comparison (lt|le|eq|ne|ge|gt) --version +# | arg: --comparison - Comparison type. Could be : `lt` (lower than), `le` (lower or equal), `eq` (equal), `ne` (not equal), `ge` (greater or equal), `gt` (greater than) +# | arg: --version - The version to compare. Need to be a version in the yunohost package version type (like `2.3.1~ynh4`) +# | ret: 0 if the evaluation is true, 1 if false. +# +# example: ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1 +# +# This helper is usually used when we need to do some actions only for some old package versions. +# +# Generally you might probably use it as follow in the upgrade script : +# ``` +# if ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1 +# then +# # Do something that is needed for the package version older than 2.3.2~ynh1 +# fi +# ``` +# +# Requires YunoHost version 3.8.0 or higher. +ynh_compare_current_package_version() { + local legacy_args=cv + declare -Ar args_array=([c]=comparison= [v]=version=) + local version + local comparison + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local current_version=$YNH_APP_CURRENT_VERSION + + # Check the syntax of the versions + if [[ ! $version =~ '~ynh' ]] || [[ ! $current_version =~ '~ynh' ]]; then + ynh_die --message="Invalid argument for version." + fi + + # Check validity of the comparator + if [[ ! $comparison =~ (lt|le|eq|ne|ge|gt) ]]; then + ynh_die --message="Invalid comparator must be : lt, le, eq, ne, ge, gt" + fi + + # Return the return value of dpkg --compare-versions + dpkg --compare-versions $current_version $comparison $version +} + +# Check if we should enforce sane default permissions (= disable rwx for 'others') +# on file/folders handled with ynh_setup_source and ynh_add_config +# +# [internal] +# +# Having a file others-readable or a folder others-executable(=enterable) +# is a security risk comparable to "chmod 777" +# +# Configuration files may contain secrets. Or even just being able to enter a +# folder may allow an attacker to do nasty stuff (maybe a file or subfolder has +# some write permission enabled for 'other' and the attacker may edit the +# content or create files as leverage for priviledge escalation ...) +# +# The sane default should be to set ownership to $app:$app. +# In specific case, you may want to set the ownership to $app:www-data +# for example if nginx needs access to static files. +# +_ynh_apply_default_permissions() { + local target=$1 + + local ynh_requirement=$(ynh_read_manifest --manifest_key="requirements.yunohost" | tr -d '<>= ') + + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then + chmod o-rwx $target + chmod g-w $target + chown -R root:root $target + if ynh_system_user_exists $app; then + chown $app:$app $target + fi + fi + + # Crons should be owned by root + # Also we don't want systemd conf, nginx conf or others stuff to be owned by the app, + # otherwise they could self-edit their own systemd conf and escalate privilege + if echo "$target" | grep -q '^/etc/cron\|/etc/php\|/etc/nginx/conf.d\|/etc/fail2ban\|/etc/systemd/system' + then + chmod 400 $target + chown root:root $target + fi +} + +int_to_bool() { + sed -e 's/^1$/True/g' -e 's/^0$/False/g' +} + +toml_to_json() { + python3 -c 'import toml, json, sys; print(json.dumps(toml.load(sys.stdin)))' +} diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE b/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE new file mode 100644 index 000000000..986360f1a --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/docker-image-extract/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2020-2023, Jeremy Lin + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md b/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md new file mode 100644 index 000000000..4c4fa301f --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/docker-image-extract/README.md @@ -0,0 +1 @@ +This is taken from https://github.com/jjlin/docker-image-extract, under MIT license. \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract b/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract new file mode 100755 index 000000000..b5dfdb7a7 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/docker-image-extract/docker-image-extract @@ -0,0 +1,288 @@ +#!/bin/sh +# +# This script pulls and extracts all files from an image in Docker Hub. +# +# Copyright (c) 2020-2023, Jeremy Lin +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +PLATFORM_DEFAULT="linux/amd64" +PLATFORM="${PLATFORM_DEFAULT}" +OUT_DIR="./output" + +usage() { + echo "This script pulls and extracts all files from an image in Docker Hub." + echo + echo "$0 [OPTIONS...] IMAGE[:REF]" + echo + echo "IMAGE can be a community user image (like 'some-user/some-image') or a" + echo "Docker official image (like 'hello-world', which contains no '/')." + echo + echo "REF is either a tag name or a full SHA-256 image digest (with a 'sha256:' prefix)." + echo "The default ref is the 'latest' tag." + echo + echo "Options:" + echo + echo " -p PLATFORM Pull image for the specified platform (default: ${PLATFORM})" + echo " For a given image on Docker Hub, the 'Tags' tab lists the" + echo " platforms supported for that image." + echo " -o OUT_DIR Extract image to the specified output dir (default: ${OUT_DIR})" + echo " -h Show help with usage examples" +} + +usage_detailed() { + usage + echo + echo "Examples:" + echo + echo "# Pull and extract all files in the 'hello-world' image tagged 'latest'." + echo "\$ $0 hello-world:latest" + echo + echo "# Same as above; ref defaults to the 'latest' tag." + echo "\$ $0 hello-world" + echo + echo "# Pull the 'hello-world' image for the 'linux/arm64/v8' platform." + echo "\$ $0 -p linux/arm64/v8 hello-world" + echo + echo "# Pull an image by digest." + echo "\$ $0 hello-world:sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042" +} + +if [ $# -eq 0 ]; then + usage_detailed + exit 0 +fi + +while getopts ':ho:p:' opt; do + case $opt in + o) + OUT_DIR="${OPTARG}" + ;; + p) + PLATFORM="${OPTARG}" + ;; + h) + usage_detailed + exit 0 + ;; + \?) + echo "ERROR: Invalid option '-$OPTARG'." + echo + usage + exit 1 + ;; + \:) echo "ERROR: Argument required for option '-$OPTARG'." + echo + usage + exit 1 + ;; + esac +done +shift $(($OPTIND - 1)) + +if [ $# -eq 0 ]; then + echo "ERROR: Image to pull must be specified." + echo + usage + exit 1 +fi + +if [ -e "${OUT_DIR}" ]; then + if [ -d "${OUT_DIR}" ]; then + echo "WARNING: Output dir already exists. If it contains a previous extracted image," + echo "there may be errors when trying to overwrite files with read-only permissions." + echo + else + echo "ERROR: Output dir already exists, but is not a directory." + exit 1 + fi +fi + +have_curl() { + command -v curl >/dev/null +} + +have_wget() { + command -v wget >/dev/null +} + +if ! have_curl && ! have_wget; then + echo "This script requires either curl or wget." + exit 1 +fi + +image_spec="$1" +image="${image_spec%%:*}" +if [ "${image#*/}" = "${image}" ]; then + # Docker official images are in the 'library' namespace. + image="library/${image}" +fi +ref="${image_spec#*:}" +if [ "${ref}" = "${image_spec}" ]; then + echo "Defaulting ref to tag 'latest'..." + ref=latest +fi + +# Split platform (OS/arch/variant) into separate variables. +# A platform specifier doesn't always include the `variant` component. +OLD_IFS="${IFS}" +IFS=/ read -r OS ARCH VARIANT <":"" (assumes key/val won't contain double quotes). + # The colon may have whitespace on either side. + grep -o "\"${key}\"[[:space:]]*:[[:space:]]*\"[^\"]\+\"" | + # Extract just by deleting the last '"', and then greedily deleting + # everything up to '"'. + sed -e 's/"$//' -e 's/.*"//' +} + +# Fetch a URL to stdout. Up to two header arguments may be specified: +# +# fetch [name1: value1] [name2: value2] +# +fetch() { + if have_curl; then + if [ $# -eq 2 ]; then + set -- -H "$2" "$1" + elif [ $# -eq 3 ]; then + set -- -H "$2" -H "$3" "$1" + fi + curl -sSL "$@" + else + if [ $# -eq 2 ]; then + set -- --header "$2" "$1" + elif [ $# -eq 3 ]; then + set -- --header "$2" --header "$3" "$1" + fi + wget -qO- "$@" + fi +} + +# https://docs.docker.com/docker-hub/api/latest/#tag/repositories +manifest_list_url="https://hub.docker.com/v2/repositories/${image}/tags/${ref}" + +# If the ref is already a SHA-256 image digest, then we don't need to look up anything. +if [ -z "${ref##sha256:*}" ]; then + digest="${ref}" +else + echo "Getting multi-arch manifest list..." + NL=' +' + digest=$(fetch "${manifest_list_url}" | + # Break up the single-line JSON output into separate lines by adding + # newlines before and after the chars '[', ']', '{', and '}'. + # This uses the \${NL} syntax because some BSD variants of sed don't + # support \n syntax in the replacement string, but instead require + # a literal newline preceded by a backslash. + sed -e 's/\([][{}]\)/\'"${NL}"'\1\'"${NL}"'/g' | + # Extract the "images":[...] list. + sed -n '/"images":/,/]/ p' | + # Each image's details are now on a separate line, e.g. + # "architecture":"arm64","features":"","variant":"v8","digest":"sha256:054c85801c4cb41511b176eb0bf13a2c4bbd41611ddd70594ec3315e88813524","os":"linux","os_features":"","os_version":null,"size":828724,"status":"active","last_pulled":"2022-09-02T22:46:48.240632Z","last_pushed":"2022-09-02T00:42:45.69226Z" + # The image details are interspersed with lines of stray punctuation, + # so grep for an arbitrary string that must be in these lines. + grep architecture | + # Search for an image that matches the platform. + while read -r image; do + # Arch is probably most likely to be unique, so check that first. + arch="$(echo ${image} | extract 'architecture')" + if [ "${arch}" != "${ARCH}" ]; then continue; fi + + os="$(echo ${image} | extract 'os')" + if [ "${os}" != "${OS}" ]; then continue; fi + + variant="$(echo ${image} | extract 'variant')" + if [ "${variant}" = "${VARIANT}" ]; then + echo ${image} | extract 'digest' + break + fi + done) + + if [ -n "${digest}" ]; then + echo "Platform ${PLATFORM} resolved to '${digest}'..." + else + echo "No image digest found. Verify that the image, ref, and platform are valid." + exit 1 + fi +fi + +# https://docs.docker.com/registry/spec/auth/token/#how-to-authenticate +api_token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" + +# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-an-image-manifest +manifest_url="https://registry-1.docker.io/v2/${image}/manifests/${digest}" + +# https://github.com/docker/distribution/blob/master/docs/spec/api.md#pulling-a-layer +blobs_base_url="https://registry-1.docker.io/v2/${image}/blobs" + +echo "Getting API token..." +token=$(fetch "${api_token_url}" | extract 'token') +auth_header="Authorization: Bearer $token" + +# https://github.com/distribution/distribution/blob/main/docs/spec/manifest-v2-2.md +docker_manifest_v2="application/vnd.docker.distribution.manifest.v2+json" + +# https://github.com/opencontainers/image-spec/blob/main/manifest.md +oci_manifest_v1="application/vnd.oci.image.manifest.v1+json" + +# Docker Hub can return either type of manifest format. Most images seem to +# use the Docker format for now, but the OCI format will likely become more +# common as features that require that format become enabled by default +# (e.g., https://github.com/docker/build-push-action/releases/tag/v3.3.0). +accept_header="Accept: ${docker_manifest_v2},${oci_manifest_v1}" + +echo "Getting image manifest for $image:$ref..." +layers=$(fetch "${manifest_url}" "${auth_header}" "${accept_header}" | + # Extract `digest` values only after the `layers` section appears. + sed -n '/"layers":/,$ p' | + extract 'digest') + +if [ -z "${layers}" ]; then + echo "No layers returned. Verify that the image and ref are valid." + exit 1 +fi + +mkdir -p "${OUT_DIR}" + +for layer in $layers; do + hash="${layer#sha256:}" + echo "Fetching and extracting layer ${hash}..." + fetch "${blobs_base_url}/${layer}" "${auth_header}" | gzip -d | tar -C "${OUT_DIR}" -xf - + # Ref: https://github.com/moby/moby/blob/master/image/spec/v1.2.md#creating-an-image-filesystem-changeset + # https://github.com/moby/moby/blob/master/pkg/archive/whiteouts.go + # Search for "whiteout" files to indicate files deleted in this layer. + OLD_IFS="${IFS}" + find "${OUT_DIR}" -name '.wh.*' | while IFS= read -r f; do + dir="${f%/*}" + wh_file="${f##*/}" + file="${wh_file#.wh.}" + # Delete both the whiteout file and the whited-out file. + rm -rf "${dir}/${wh_file}" "${dir}/${file}" + done + IFS="${OLD_IFS}" +done + +echo "Image contents extracted into ${OUT_DIR}." \ No newline at end of file diff --git a/helpers/helpers.v2.1.d/vendor/n/LICENSE b/helpers/helpers.v2.1.d/vendor/n/LICENSE new file mode 100644 index 000000000..8e04e8467 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/n/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 TJ Holowaychuk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/helpers/helpers.v2.1.d/vendor/n/README.md b/helpers/helpers.v2.1.d/vendor/n/README.md new file mode 100644 index 000000000..9a29a3936 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/n/README.md @@ -0,0 +1 @@ +This is taken from https://github.com/tj/n/ diff --git a/helpers/helpers.v2.1.d/vendor/n/n b/helpers/helpers.v2.1.d/vendor/n/n new file mode 100755 index 000000000..86b6a0fa9 --- /dev/null +++ b/helpers/helpers.v2.1.d/vendor/n/n @@ -0,0 +1,1713 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2155 +# Disabled "Declare and assign separately to avoid masking return values": https://github.com/koalaman/shellcheck/wiki/SC2155 + +# +# log +# + +log() { + printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" +} + +# +# verbose_log +# Can suppress with --quiet. +# Like log but to stderr rather than stdout, so can also be used from "display" routines. +# + +verbose_log() { + if [[ "${SHOW_VERBOSE_LOG}" == "true" ]]; then + >&2 printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" + fi +} + +# +# Exit with the given +# + +abort() { + >&2 printf "\n ${SGR_RED}Error: %s${SGR_RESET}\n\n" "$*" && exit 1 +} + +# +# Synopsis: trace message ... +# Debugging output to stderr, not used in production code. +# + +function trace() { + >&2 printf "trace: %s\n" "$*" +} + +# +# Synopsis: echo_red message ... +# Highlight message in colour (on stdout). +# + +function echo_red() { + printf "${SGR_RED}%s${SGR_RESET}\n" "$*" +} + +# +# Synopsis: n_grep +# grep wrapper to ensure consistent grep options and circumvent aliases. +# + +function n_grep() { + GREP_OPTIONS='' command grep "$@" +} + +# +# Setup and state +# + +VERSION="v9.2.3" + +N_PREFIX="${N_PREFIX-/usr/local}" +N_PREFIX=${N_PREFIX%/} +readonly N_PREFIX + +N_CACHE_PREFIX="${N_CACHE_PREFIX-${N_PREFIX}}" +N_CACHE_PREFIX=${N_CACHE_PREFIX%/} +CACHE_DIR="${N_CACHE_PREFIX}/n/versions" +readonly N_CACHE_PREFIX CACHE_DIR + +N_NODE_MIRROR=${N_NODE_MIRROR:-${NODE_MIRROR:-https://nodejs.org/dist}} +N_NODE_MIRROR=${N_NODE_MIRROR%/} +readonly N_NODE_MIRROR + +N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR:-https://nodejs.org/download} +N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR%/} +readonly N_NODE_DOWNLOAD_MIRROR + +# Using xz instead of gzip is enabled by default, if xz compatibility checks pass. +# User may set N_USE_XZ to 0 to disable, or set to anything else to enable. +# May also be overridden by command line flags. + +# Normalise external values to true/false +if [[ "${N_USE_XZ}" = "0" ]]; then + N_USE_XZ="false" +elif [[ -n "${N_USE_XZ+defined}" ]]; then + N_USE_XZ="true" +fi +# Not setting to readonly. Overriden by CLI flags, and update_xz_settings_for_version. + +N_MAX_REMOTE_MATCHES=${N_MAX_REMOTE_MATCHES:-20} +# modified by update_mirror_settings_for_version +g_mirror_url=${N_NODE_MIRROR} +g_mirror_folder_name="node" + +# Options for curl and wget. +# Defining commands in variables is fraught (https://mywiki.wooledge.org/BashFAQ/050) +# but we can follow the simple case and store arguments in an array. + +GET_SHOWS_PROGRESS="false" +# --location to follow redirects +# --fail to avoid happily downloading error page from web server for 404 et al +# --show-error to show why failed (on stderr) +CURL_OPTIONS=( "--location" "--fail" "--show-error" ) +if [[ -t 1 ]]; then + CURL_OPTIONS+=( "--progress-bar" ) + command -v curl &> /dev/null && GET_SHOWS_PROGRESS="true" +else + CURL_OPTIONS+=( "--silent" ) +fi +WGET_OPTIONS=( "-q" "-O-" ) + +# Legacy support using unprefixed env. No longer documented in README. +if [ -n "$HTTP_USER" ];then + if [ -z "$HTTP_PASSWORD" ]; then + abort "Must specify HTTP_PASSWORD when supplying HTTP_USER" + fi + CURL_OPTIONS+=( "-u $HTTP_USER:$HTTP_PASSWORD" ) + WGET_OPTIONS+=( "--http-password=$HTTP_PASSWORD" + "--http-user=$HTTP_USER" ) +elif [ -n "$HTTP_PASSWORD" ]; then + abort "Must specify HTTP_USER when supplying HTTP_PASSWORD" +fi + +# Set by set_active_node +g_active_node= + +# set by various lookups to allow mixed logging and return value from function, especially for engine and node +g_target_node= + +DOWNLOAD=false # set to opt-out of activate (install), and opt-in to download (run, exec) +ARCH= +SHOW_VERBOSE_LOG="true" +OFFLINE=false + +# ANSI escape codes +# https://en.wikipedia.org/wiki/ANSI_escape_code +# https://no-color.org +# https://bixense.com/clicolors + +USE_COLOR="true" +if [[ -n "${CLICOLOR_FORCE+defined}" && "${CLICOLOR_FORCE}" != "0" ]]; then + USE_COLOR="true" +elif [[ -n "${NO_COLOR+defined}" || "${CLICOLOR}" = "0" || ! -t 1 ]]; then + USE_COLOR="false" +fi +readonly USE_COLOR +# Select Graphic Rendition codes +if [[ "${USE_COLOR}" = "true" ]]; then + # KISS and use codes rather than tput, avoid dealing with missing tput or TERM. + readonly SGR_RESET="\033[0m" + readonly SGR_FAINT="\033[2m" + readonly SGR_RED="\033[31m" + readonly SGR_CYAN="\033[36m" +else + readonly SGR_RESET= + readonly SGR_FAINT= + readonly SGR_RED= + readonly SGR_CYAN= +fi + +# +# set_arch to override $(uname -a) +# + +set_arch() { + if test -n "$1"; then + ARCH="$1" + else + abort "missing -a|--arch value" + fi +} + +# +# Synopsis: set_insecure +# Globals modified: +# - CURL_OPTIONS +# - WGET_OPTIONS +# + +function set_insecure() { + CURL_OPTIONS+=( "--insecure" ) + WGET_OPTIONS+=( "--no-check-certificate" ) +} + +# +# Synposis: display_major_version numeric-version +# +display_major_version() { + local version=$1 + version="${version#v}" + version="${version%%.*}" + echo "${version}" +} + +# +# Synopsis: update_mirror_settings_for_version version +# e.g. means using download mirror and folder is nightly +# Globals modified: +# - g_mirror_url +# - g_mirror_folder_name +# + +function update_mirror_settings_for_version() { + if is_download_folder "$1" ; then + g_mirror_folder_name="$1" + g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" + elif is_download_version "$1"; then + [[ "$1" =~ ^([^/]+)/(.*) ]] + local remote_folder="${BASH_REMATCH[1]}" + g_mirror_folder_name="${remote_folder}" + g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" + fi +} + +# +# Synopsis: update_xz_settings_for_version numeric-version +# Globals modified: +# - N_USE_XZ +# + +function update_xz_settings_for_version() { + # tarballs in xz format were available in later version of iojs, but KISS and only use xz from v4. + if [[ "${N_USE_XZ}" = "true" ]]; then + local major_version="$(display_major_version "$1")" + if [[ "${major_version}" -lt 4 ]]; then + N_USE_XZ="false" + fi + fi +} + +# +# Synopsis: update_arch_settings_for_version numeric-version +# Globals modified: +# - ARCH +# + +function update_arch_settings_for_version() { + local tarball_platform="$(display_tarball_platform)" + if [[ -z "${ARCH}" && "${tarball_platform}" = "darwin-arm64" ]]; then + # First native builds were for v16, but can use x64 in rosetta for older versions. + local major_version="$(display_major_version "$1")" + if [[ "${major_version}" -lt 16 ]]; then + ARCH=x64 + fi + fi +} + +# +# Synopsis: is_lts_codename version +# + +function is_lts_codename() { + # https://github.com/nodejs/Release/blob/master/CODENAMES.md + # e.g. argon, Boron + [[ "$1" =~ ^([Aa]rgon|[Bb]oron|[Cc]arbon|[Dd]ubnium|[Ee]rbium|[Ff]ermium|[Gg]allium|[Hh]ydrogen|[Ii]ron|[Jj]od)$ ]] +} + +# +# Synopsis: is_download_folder version +# + +function is_download_folder() { + # e.g. nightly + [[ "$1" =~ ^(next-nightly|nightly|rc|release|test|v8-canary)$ ]] +} + +# +# Synopsis: is_download_version version +# + +function is_download_version() { + # e.g. nightly/, nightly/latest, nightly/v11 + if [[ "$1" =~ ^([^/]+)/(.*) ]]; then + local remote_folder="${BASH_REMATCH[1]}" + is_download_folder "${remote_folder}" + return + fi + return 2 +} + +# +# Synopsis: is_numeric_version version +# + +function is_numeric_version() { + # e.g. 6, v7.1, 8.11.3 + [[ "$1" =~ ^[v]{0,1}[0-9]+(\.[0-9]+){0,2}$ ]] +} + +# +# Synopsis: is_exact_numeric_version version +# + +function is_exact_numeric_version() { + # e.g. 6, v7.1, 8.11.3 + [[ "$1" =~ ^[v]{0,1}[0-9]+\.[0-9]+\.[0-9]+$ ]] +} + +# +# Synopsis: is_node_support_version version +# Reference: https://github.com/nodejs/package-maintenance/issues/236#issue-474783582 +# + +function is_node_support_version() { + [[ "$1" =~ ^(active|lts_active|lts_latest|lts|current|supported)$ ]] +} + +# +# Synopsis: display_latest_node_support_alias version +# Map aliases onto existing n aliases, current and lts +# + +function display_latest_node_support_alias() { + case "$1" in + "active") printf "current" ;; + "lts_active") printf "lts" ;; + "lts_latest") printf "lts" ;; + "lts") printf "lts" ;; + "current") printf "current" ;; + "supported") printf "current" ;; + *) printf "unexpected-version" + esac +} + +# +# Functions used when showing versions installed +# + +enter_fullscreen() { + # Set cursor to be invisible + tput civis 2> /dev/null + # Save screen contents + tput smcup 2> /dev/null + stty -echo +} + +leave_fullscreen() { + # Set cursor to normal + tput cnorm 2> /dev/null + # Restore screen contents + tput rmcup 2> /dev/null + stty echo +} + +handle_sigint() { + leave_fullscreen + S="$?" + kill 0 + exit $S +} + +handle_sigtstp() { + leave_fullscreen + kill -s SIGSTOP $$ +} + +# +# Output usage information. +# + +display_help() { + cat <<-EOF + +Usage: n [options] [COMMAND] [args] + +Commands: + + n Display downloaded Node.js versions and install selection + n latest Install the latest Node.js release (downloading if necessary) + n lts Install the latest LTS Node.js release (downloading if necessary) + n Install Node.js (downloading if necessary) + n install Install Node.js (downloading if necessary) + n run [args ...] Execute downloaded Node.js with [args ...] + n which Output path for downloaded node + n exec [args...] Execute command with modified PATH, so downloaded node and npm first + n rm Remove the given downloaded version(s) + n prune Remove all downloaded versions except the installed version + n --latest Output the latest Node.js version available + n --lts Output the latest LTS Node.js version available + n ls Output downloaded versions + n ls-remote [version] Output matching versions available for download + n uninstall Remove the installed Node.js + +Options: + + -V, --version Output version of n + -h, --help Display help information + -p, --preserve Preserve npm and npx during install of Node.js + -q, --quiet Disable curl output. Disable log messages processing "auto" and "engine" labels. + -d, --download Download if necessary, and don't make active + -a, --arch Override system architecture + --offline Resolve target version against cached downloads instead of internet lookup + --all ls-remote displays all matches instead of last 20 + --insecure Turn off certificate checking for https requests (may be needed from behind a proxy server) + --use-xz/--no-use-xz Override automatic detection of xz support and enable/disable use of xz compressed node downloads. + +Aliases: + + install: i + latest: current + ls: list + lsr: ls-remote + lts: stable + rm: - + run: use, as + which: bin + +Versions: + + Numeric version numbers can be complete or incomplete, with an optional leading 'v'. + Versions can also be specified by label, or codename, + and other downloadable releases by / + + 4.9.1, 8, v6.1 Numeric versions + lts Newest Long Term Support official release + latest, current Newest official release + auto Read version from file: .n-node-version, .node-version, .nvmrc, or package.json + engine Read version from package.json + boron, carbon Codenames for release streams + lts_latest Node.js support aliases + + and nightly, rc/10 et al + +EOF +} + +err_no_installed_print_help() { + display_help + abort "no downloaded versions yet, see above help for commands" +} + +# +# Synopsis: next_version_installed selected_version +# Output version after selected (which may be blank under some circumstances). +# + +function next_version_installed() { + display_cache_versions | n_grep "$1" -A 1 | tail -n 1 +} + +# +# Synopsis: prev_version_installed selected_version +# Output version before selected (which may be blank under some circumstances). +# + +function prev_version_installed() { + display_cache_versions | n_grep "$1" -B 1 | head -n 1 +} + +# +# Output n version. +# + +display_n_version() { + echo "$VERSION" && exit 0 +} + +# +# Synopsis: set_active_node +# Checks cached downloads for a binary matching the active node. +# Globals modified: +# - g_active_node +# + +function set_active_node() { + g_active_node= + local node_path="$(command -v node)" + if [[ -x "${node_path}" ]]; then + local installed_version=$(node --version) + installed_version=${installed_version#v} + for dir in "${CACHE_DIR}"/*/ ; do + local folder_name="${dir%/}" + folder_name="${folder_name##*/}" + if diff &> /dev/null \ + "${CACHE_DIR}/${folder_name}/${installed_version}/bin/node" \ + "${node_path}" ; then + g_active_node="${folder_name}/${installed_version}" + break + fi + done + fi +} + +# +# Display sorted versions directories paths. +# + +display_versions_paths() { + find "$CACHE_DIR" -maxdepth 2 -type d \ + | sed 's|'"$CACHE_DIR"'/||g' \ + | n_grep -E "/[0-9]+\.[0-9]+\.[0-9]+" \ + | sed 's|/|.|' \ + | sort -k 1,1 -k 2,2n -k 3,3n -k 4,4n -t . \ + | sed 's|\.|/|' +} + +# +# Display installed versions with +# + +display_versions_with_selected() { + local selected="$1" + echo + for version in $(display_versions_paths); do + if test "$version" = "$selected"; then + printf " ${SGR_CYAN}ο${SGR_RESET} %s\n" "$version" + else + printf " ${SGR_FAINT}%s${SGR_RESET}\n" "$version" + fi + done + echo + printf "Use up/down arrow keys to select a version, return key to install, d to delete, q to quit" +} + +# +# Synopsis: display_cache_versions +# + +function display_cache_versions() { + for folder_and_version in $(display_versions_paths); do + echo "${folder_and_version}" + done +} + +# +# Display current node --version and others installed. +# + +menu_select_cache_versions() { + enter_fullscreen + set_active_node + local selected="${g_active_node}" + + clear + display_versions_with_selected "${selected}" + + trap handle_sigint INT + trap handle_sigtstp SIGTSTP + + ESCAPE_SEQ=$'\033' + UP=$'A' + DOWN=$'B' + CTRL_P=$'\020' + CTRL_N=$'\016' + + while true; do + read -rsn 1 key + case "$key" in + "$ESCAPE_SEQ") + # Handle ESC sequences followed by other characters, i.e. arrow keys + read -rsn 1 -t 1 tmp + # See "[" if terminal in normal mode, and "0" in application mode + if [[ "$tmp" == "[" || "$tmp" == "O" ]]; then + read -rsn 1 -t 1 arrow + case "$arrow" in + "$UP") + clear + selected="$(prev_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + "$DOWN") + clear + selected="$(next_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + esac + fi + ;; + "d") + if [[ -n "${selected}" ]]; then + clear + # Note: prev/next is constrained to min/max + local after_delete_selection="$(next_version_installed "${selected}")" + if [[ "${after_delete_selection}" == "${selected}" ]]; then + after_delete_selection="$(prev_version_installed "${selected}")" + fi + remove_versions "${selected}" + + if [[ "${after_delete_selection}" == "${selected}" ]]; then + clear + leave_fullscreen + echo "All downloaded versions have been deleted from cache." + exit + fi + + selected="${after_delete_selection}" + display_versions_with_selected "${selected}" + fi + ;; + # Vim or Emacs 'up' key + "k"|"$CTRL_P") + clear + selected="$(prev_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + # Vim or Emacs 'down' key + "j"|"$CTRL_N") + clear + selected="$(next_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + "q") + clear + leave_fullscreen + exit + ;; + "") + # enter key returns empty string + leave_fullscreen + [[ -n "${selected}" ]] && activate "${selected}" + exit + ;; + esac + done +} + +# +# Move up a line and erase. +# + +erase_line() { + printf "\033[1A\033[2K" +} + +# +# Disable PaX mprotect for +# + +disable_pax_mprotect() { + test -z "$1" && abort "binary required" + local binary="$1" + + # try to disable mprotect via XATTR_PAX header + local PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl-ng 2>&1)" + local PAXCTL_ERROR=1 + if [ -x "$PAXCTL" ]; then + $PAXCTL -l && $PAXCTL -m "$binary" >/dev/null 2>&1 + PAXCTL_ERROR="$?" + fi + + # try to disable mprotect via PT_PAX header + if [ "$PAXCTL_ERROR" != 0 ]; then + PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl 2>&1)" + if [ -x "$PAXCTL" ]; then + $PAXCTL -Cm "$binary" >/dev/null 2>&1 + fi + fi +} + +# +# clean_copy_folder +# + +clean_copy_folder() { + local source="$1" + local target="$2" + if [[ -d "${source}" ]]; then + rm -rf "${target}" + cp -fR "${source}" "${target}" + fi +} + +# +# Activate +# + +activate() { + local version="$1" + local dir="$CACHE_DIR/$version" + local original_node="$(command -v node)" + local installed_node="${N_PREFIX}/bin/node" + log "copying" "$version" + + + # Ideally we would just copy from cache to N_PREFIX, but there are some complications + # - various linux versions use symlinks for folders in /usr/local and also error when copy folder onto symlink + # - we have used cp for years, so keep using it for backwards compatibility (instead of say rsync) + # - we allow preserving npm + # - we want to be somewhat robust to changes in tarball contents, so use find instead of hard-code expected subfolders + # + # This code was purist and concise for a long time. + # Now twice as much code, but using same code path for all uses, and supporting more setups. + + # Copy lib before bin so symlink targets exist. + # lib + mkdir -p "$N_PREFIX/lib" + # Copy everything except node_modules. + find "$dir/lib" -mindepth 1 -maxdepth 1 \! -name node_modules -exec cp -fR "{}" "$N_PREFIX/lib" \; + if [[ -z "${N_PRESERVE_NPM}" ]]; then + mkdir -p "$N_PREFIX/lib/node_modules" + # Copy just npm, skipping possible added global modules after download. Clean copy to avoid version change problems. + clean_copy_folder "$dir/lib/node_modules/npm" "$N_PREFIX/lib/node_modules/npm" + fi + # Takes same steps for corepack (experimental in node 16.9.0) as for npm, to avoid version problems. + if [[ -e "$dir/lib/node_modules/corepack" && -z "${N_PRESERVE_COREPACK}" ]]; then + mkdir -p "$N_PREFIX/lib/node_modules" + clean_copy_folder "$dir/lib/node_modules/corepack" "$N_PREFIX/lib/node_modules/corepack" + fi + + # bin + mkdir -p "$N_PREFIX/bin" + # Remove old node to avoid potential problems with firewall getting confused on Darwin by overwrite. + rm -f "$N_PREFIX/bin/node" + # Copy bin items by hand, in case user has installed global npm modules into cache. + cp -f "$dir/bin/node" "$N_PREFIX/bin" + [[ -e "$dir/bin/node-waf" ]] && cp -f "$dir/bin/node-waf" "$N_PREFIX/bin" # v0.8.x + if [[ -z "${N_PRESERVE_COREPACK}" ]]; then + [[ -e "$dir/bin/corepack" ]] && cp -fR "$dir/bin/corepack" "$N_PREFIX/bin" # from 16.9.0 + fi + if [[ -z "${N_PRESERVE_NPM}" ]]; then + [[ -e "$dir/bin/npm" ]] && cp -fR "$dir/bin/npm" "$N_PREFIX/bin" + [[ -e "$dir/bin/npx" ]] && cp -fR "$dir/bin/npx" "$N_PREFIX/bin" + fi + + # include + mkdir -p "$N_PREFIX/include" + find "$dir/include" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/include" \; + + # share + mkdir -p "$N_PREFIX/share" + # Copy everything except man, at it is a symlink on some Linux (e.g. archlinux). + find "$dir/share" -mindepth 1 -maxdepth 1 \! -name man -exec cp -fR "{}" "$N_PREFIX/share" \; + mkdir -p "$N_PREFIX/share/man" + find "$dir/share/man" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/share/man" \; + + disable_pax_mprotect "${installed_node}" + + local active_node="$(command -v node)" + if [[ -e "${active_node}" && -e "${installed_node}" && "${active_node}" != "${installed_node}" ]]; then + # Installed and active are different which might be a PATH problem. List both to give user some clues. + log "installed" "$("${installed_node}" --version) to ${installed_node}" + log "active" "$("${active_node}" --version) at ${active_node}" + else + local npm_version_str="" + local installed_npm="${N_PREFIX}/bin/npm" + local active_npm="$(command -v npm)" + if [[ -z "${N_PRESERVE_NPM}" && -e "${active_npm}" && -e "${installed_npm}" && "${active_npm}" = "${installed_npm}" ]]; then + npm_version_str=" (with npm $(npm --version))" + fi + + log "installed" "$("${installed_node}" --version)${npm_version_str}" + + # Extra tips for changed location. + if [[ -e "${active_node}" && -e "${original_node}" && "${active_node}" != "${original_node}" ]]; then + printf '\nNote: the node command changed location and the old location may be remembered in your current shell.\n' + log old "${original_node}" + log new "${active_node}" + printf 'If "node --version" shows the old version then start a new shell, or reset the location hash with:\nhash -r (for bash, zsh, ash, dash, and ksh)\nrehash (for csh and tcsh)\n' + fi + fi +} + +# +# Install +# + +install() { + [[ -z "$1" ]] && abort "version required" + local version + get_latest_resolved_version "$1" || return 2 + version="${g_target_node}" + [[ -n "${version}" ]] || abort "no version found for '$1'" + update_mirror_settings_for_version "$1" + update_xz_settings_for_version "${version}" + update_arch_settings_for_version "${version}" + + local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + + # Note: decompression flags ignored with default Darwin tar which autodetects. + if test "$N_USE_XZ" = "true"; then + local tarflag="-Jx" + else + local tarflag="-zx" + fi + + if test -d "$dir"; then + if [[ ! -e "$dir/n.lock" ]] ; then + if [[ "$DOWNLOAD" == "false" ]] ; then + activate "${g_mirror_folder_name}/${version}" + fi + exit + fi + fi + if [[ "$OFFLINE" == "true" ]]; then + abort "version unavailable offline" + fi + + log installing "${g_mirror_folder_name}-v$version" + + local url="$(tarball_url "$version")" + is_ok "${url}" || abort "download preflight failed for '$version' (${url})" + + log mkdir "$dir" + mkdir -p "$dir" || abort "sudo required (or change ownership, or define N_PREFIX)" + touch "$dir/n.lock" + + cd "${dir}" || abort "Failed to cd to ${dir}" + + log fetch "$url" + do_get "${url}" | tar "$tarflag" --strip-components=1 --no-same-owner -f - + pipe_results=( "${PIPESTATUS[@]}" ) + if [[ "${pipe_results[0]}" -ne 0 ]]; then + abort "failed to download archive for $version" + fi + if [[ "${pipe_results[1]}" -ne 0 ]]; then + abort "failed to extract archive for $version" + fi + [ "$GET_SHOWS_PROGRESS" = "true" ] && erase_line + rm -f "$dir/n.lock" + + disable_pax_mprotect bin/node + + if [[ "$DOWNLOAD" == "false" ]]; then + activate "${g_mirror_folder_name}/$version" + fi +} + +# +# Be more silent. +# + +set_quiet() { + SHOW_VERBOSE_LOG="false" + command -v curl > /dev/null && CURL_OPTIONS+=( "--silent" ) && GET_SHOWS_PROGRESS="false" +} + +# +# Synopsis: do_get [option...] url +# Call curl or wget with combination of global and passed options. +# + +function do_get() { + if command -v curl &> /dev/null; then + curl "${CURL_OPTIONS[@]}" "$@" + elif command -v wget &> /dev/null; then + wget "${WGET_OPTIONS[@]}" "$@" + else + abort "curl or wget command required" + fi +} + +# +# Synopsis: do_get_index [option...] url +# Call curl or wget with combination of global and passed options, +# with options tweaked to be more suitable for getting index. +# + +function do_get_index() { + if command -v curl &> /dev/null; then + # --silent to suppress progress et al + curl --silent "${CURL_OPTIONS[@]}" "$@" + elif command -v wget &> /dev/null; then + wget "${WGET_OPTIONS[@]}" "$@" + else + abort "curl or wget command required" + fi +} + +# +# Synopsis: remove_versions version ... +# + +function remove_versions() { + [[ -z "$1" ]] && abort "version(s) required" + while [[ $# -ne 0 ]]; do + local version + get_latest_resolved_version "$1" || break + version="${g_target_node}" + if [[ -n "${version}" ]]; then + update_mirror_settings_for_version "$1" + local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + if [[ -s "${dir}" ]]; then + rm -rf "${dir}" + else + echo "$1 (${version}) not in downloads cache" + fi + else + echo "No version found for '$1'" + fi + shift + done +} + +# +# Synopsis: prune_cache +# + +function prune_cache() { + set_active_node + + for folder_and_version in $(display_versions_paths); do + if [[ "${folder_and_version}" != "${g_active_node}" ]]; then + echo "${folder_and_version}" + rm -rf "${CACHE_DIR:?}/${folder_and_version}" + fi + done +} + +# +# Synopsis: find_cached_version version +# Finds cache directory for resolved version. +# Globals modified: +# - g_cached_version + +function find_cached_version() { + [[ -z "$1" ]] && abort "version required" + local version + get_latest_resolved_version "$1" || exit 1 + version="${g_target_node}" + [[ -n "${version}" ]] || abort "no version found for '$1'" + + update_mirror_settings_for_version "$1" + g_cached_version="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + if [[ ! -d "${g_cached_version}" && "${DOWNLOAD}" == "true" ]]; then + (install "${version}") + fi + [[ -d "${g_cached_version}" ]] || abort "'$1' (${version}) not in downloads cache" +} + + +# +# Synopsis: display_bin_path_for_version version +# + +function display_bin_path_for_version() { + find_cached_version "$1" + echo "${g_cached_version}/bin/node" +} + +# +# Synopsis: run_with_version version [args...] +# Run the given of node with [args ..] +# + +function run_with_version() { + find_cached_version "$1" + shift # remove version from parameters + exec "${g_cached_version}/bin/node" "$@" +} + +# +# Synopsis: exec_with_version command [args...] +# Modify the path to include and execute command. +# + +function exec_with_version() { + find_cached_version "$1" + shift # remove version from parameters + PATH="${g_cached_version}/bin:$PATH" exec "$@" +} + +# +# Synopsis: is_ok url +# Check the HEAD response of . +# + +function is_ok() { + # Note: both curl and wget can follow redirects, as present on some mirrors (e.g. https://npm.taobao.org/mirrors/node). + # The output is complicated with redirects, so keep it simple and use command status rather than parse output. + if command -v curl &> /dev/null; then + do_get --silent --head "$1" > /dev/null || return 1 + else + do_get --spider "$1" > /dev/null || return 1 + fi +} + +# +# Synopsis: can_use_xz +# Test system to see if xz decompression is supported by tar. +# + +function can_use_xz() { + # Be conservative and only enable if xz is likely to work. Unfortunately we can't directly query tar itself. + # For research, see https://github.com/shadowspawn/nvh/issues/8 + local uname_s="$(uname -s)" + if [[ "${uname_s}" = "Linux" ]] && command -v xz &> /dev/null ; then + # tar on linux is likely to support xz if it is available as a command + return 0 + elif [[ "${uname_s}" = "Darwin" ]]; then + local macos_version="$(sw_vers -productVersion)" + local macos_major_version="$(echo "${macos_version}" | cut -d '.' -f 1)" + local macos_minor_version="$(echo "${macos_version}" | cut -d '.' -f 2)" + if [[ "${macos_major_version}" -gt 10 || "${macos_minor_version}" -gt 8 ]]; then + # tar on recent Darwin has xz support built-in + return 0 + fi + fi + return 2 # not supported +} + +# +# Synopsis: display_tarball_platform +# + +function display_tarball_platform() { + # https://en.wikipedia.org/wiki/Uname + + local os="unexpected_os" + local uname_a="$(uname -a)" + case "${uname_a}" in + Linux*) os="linux" ;; + Darwin*) os="darwin" ;; + SunOS*) os="sunos" ;; + AIX*) os="aix" ;; + CYGWIN*) >&2 echo_red "Cygwin is not supported by n" ;; + MINGW*) >&2 echo_red "Git BASH (MSYS) is not supported by n" ;; + esac + + local arch="unexpected_arch" + local uname_m="$(uname -m)" + case "${uname_m}" in + x86_64) arch=x64 ;; + i386 | i686) arch="x86" ;; + aarch64) arch=arm64 ;; + armv8l) arch=arm64 ;; # armv8l probably supports arm64, and there is no specific armv8l build so give it a go + *) + # e.g. armv6l, armv7l, arm64 + arch="${uname_m}" + ;; + esac + # Override from command line, or version specific adjustment. + [ -n "$ARCH" ] && arch="$ARCH" + + echo "${os}-${arch}" +} + +# +# Synopsis: display_compatible_file_field +# display for current platform, as per field in index.tab, which is different than actual download +# + +function display_compatible_file_field { + local compatible_file_field="$(display_tarball_platform)" + if [[ -z "${ARCH}" && "${compatible_file_field}" = "darwin-arm64" ]]; then + # Look for arm64 for native but also x64 for older versions which can run in rosetta. + # (Downside is will get an install error if install version above 16 with x64 and not arm64.) + compatible_file_field="osx-arm64-tar|osx-x64-tar" + elif [[ "${compatible_file_field}" =~ darwin-(.*) ]]; then + compatible_file_field="osx-${BASH_REMATCH[1]}-tar" + fi + echo "${compatible_file_field}" +} + +# +# Synopsis: tarball_url version +# + +function tarball_url() { + local version="$1" + local ext=gz + [ "$N_USE_XZ" = "true" ] && ext="xz" + echo "${g_mirror_url}/v${version}/node-v${version}-$(display_tarball_platform).tar.${ext}" +} + +# +# Synopsis: get_file_node_version filename +# Sets g_target_node +# + +function get_file_node_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + # read returns a non-zero status but does still work if there is no line ending + local version + <"${filepath}" read -r version + # trim possible trailing \d from a Windows created file + version="${version%%[[:space:]]}" + verbose_log "read" "${version}" + g_target_node="${version}" +} + +# +# Synopsis: get_package_engine_version\ +# Sets g_target_node +# + +function get_package_engine_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + command -v node &> /dev/null || abort "an active version of node is required to read 'engines' from package.json" + local range + range="$(node -e "package = require('${filepath}'); if (package && package.engines && package.engines.node) console.log(package.engines.node)")" + verbose_log "read" "${range}" + [[ -n "${range}" ]] || return 2 + if [[ "*" == "${range}" ]]; then + verbose_log "target" "current" + g_target_node="current" + return + fi + + local version + if [[ "${range}" =~ ^([>~^=]|\>\=)?v?([0-9]+(\.[0-9]+){0,2})(.[xX*])?$ ]]; then + local operator="${BASH_REMATCH[1]}" + version="${BASH_REMATCH[2]}" + case "${operator}" in + '' | =) ;; + \> | \>=) version="current" ;; + \~) [[ "${version}" =~ ^([0-9]+\.[0-9]+)\.[0-9]+$ ]] && version="${BASH_REMATCH[1]}" ;; + ^) [[ "${version}" =~ ^([0-9]+) ]] && version="${BASH_REMATCH[1]}" ;; + esac + verbose_log "target" "${version}" + else + command -v npx &> /dev/null || abort "an active version of npx is required to use complex 'engine' ranges from package.json" + [[ "$OFFLINE" != "true" ]] || abort "offline: an internet connection is required for looking up complex 'engine' ranges from package.json" + verbose_log "resolving" "${range}" + local version_per_line="$(n lsr --all)" + local versions_one_line=$(echo "${version_per_line}" | tr '\n' ' ') + # Using semver@7 so works with older versions of node. + # shellcheck disable=SC2086 + version=$(npm_config_yes=true npx --quiet semver@7 -r "${range}" ${versions_one_line} | tail -n 1) + fi + g_target_node="${version}" +} + +# +# Synopsis: get_nvmrc_version +# Sets g_target_node +# + +function get_nvmrc_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + local version + <"${filepath}" read -r version + verbose_log "read" "${version}" + # Translate from nvm aliases + case "${version}" in + lts/\*) version="lts" ;; + lts/*) version="${version:4}" ;; + node) version="current" ;; + *) ;; + esac + g_target_node="${version}" +} + +# +# Synopsis: get_engine_version [error-message] +# Sets g_target_node +# + +function get_engine_version() { + g_target_node= + local error_message="${1-package.json not found}" + local parent + parent="${PWD}" + while [[ -n "${parent}" ]]; do + if [[ -e "${parent}/package.json" ]]; then + get_package_engine_version "${parent}/package.json" + else + parent=${parent%/*} + continue + fi + break + done + [[ -n "${parent}" ]] || abort "${error_message}" + [[ -n "${g_target_node}" ]] || abort "did not find supported version of node in 'engines' field of package.json" +} + +# +# Synopsis: get_auto_version +# Sets g_target_node +# + +function get_auto_version() { + g_target_node= + # Search for a version control file first + local parent + parent="${PWD}" + while [[ -n "${parent}" ]]; do + if [[ -e "${parent}/.n-node-version" ]]; then + get_file_node_version "${parent}/.n-node-version" + elif [[ -e "${parent}/.node-version" ]]; then + get_file_node_version "${parent}/.node-version" + elif [[ -e "${parent}/.nvmrc" ]]; then + get_nvmrc_version "${parent}/.nvmrc" + else + parent=${parent%/*} + continue + fi + break + done + # Fallback to package.json + [[ -n "${parent}" ]] || get_engine_version "no file found for auto version (.n-node-version, .node-version, .nvmrc, or package.json)" + [[ -n "${g_target_node}" ]] || abort "file found for auto did not contain target version of node" +} + +# +# Synopsis: get_latest_resolved_version version +# Sets g_target_node +# + +function get_latest_resolved_version() { + g_target_node= + local version=${1} + simple_version=${version#node/} # Only place supporting node/ [sic] + if is_exact_numeric_version "${simple_version}"; then + # Just numbers, already resolved, no need to lookup first. + simple_version="${simple_version#v}" + g_target_node="${simple_version}" + elif [[ "$OFFLINE" == "true" ]]; then + g_target_node=$(display_local_versions "${version}") + else + # Complicated recognising exact version, KISS and lookup. + g_target_node=$(N_MAX_REMOTE_MATCHES=1 display_remote_versions "$version") + fi +} + +# +# Synopsis: display_remote_index +# index.tab reference: https://github.com/nodejs/nodejs-dist-indexer +# Index fields are: version date files npm v8 uv zlib openssl modules lts security +# KISS and just return fields we currently care about: version files lts +# + +display_remote_index() { + local index_url="${g_mirror_url}/index.tab" + # tail to remove header line + do_get_index "${index_url}" | tail -n +2 | cut -f 1,3,10 + if [[ "${PIPESTATUS[0]}" -ne 0 ]]; then + # Reminder: abort will only exit subshell, but consistent error display + abort "failed to download version index (${index_url})" + fi +} + +# +# Synopsis: display_match_limit limit +# + +function display_match_limit(){ + if [[ "$1" -gt 1 && "$1" -lt 32000 ]]; then + echo "Listing remote... Displaying $1 matches (use --all to see all)." + fi +} + +# +# Synopsis: display_local_versions version +# + +function display_local_versions() { + local version="$1" + local match='.' + verbose_log "offline" "matching cached versions" + + # Transform some labels before processing further. + if is_node_support_version "${version}"; then + version="$(display_latest_node_support_alias "${version}")" + match_count=1 + elif [[ "${version}" = "auto" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_auto_version || return 2 + version="${g_target_node}" + elif [[ "${version}" = "engine" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_engine_version || return 2 + version="${g_target_node}" + fi + + if [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match='^node/.' + elif is_exact_numeric_version "${version}"; then + # Quote any dots in version so they are literal for expression + match="^node/${version//\./\.}" + elif is_numeric_version "${version}"; then + version="${version#v}" + # Quote any dots in version so they are literal for expression + match="${version//\./\.}" + # Avoid 1.2 matching 1.23 + match="^node/${match}[^0-9]" + # elif is_lts_codename "${version}"; then + # see if demand + elif is_download_folder "${version}"; then + match="^${version}/" + # elif is_download_version "${version}"; then + # see if demand + else + abort "invalid version '$1' for offline matching" + fi + + display_versions_paths \ + | n_grep -E "${match}" \ + | tail -n 1 \ + | sed 's|node/||' +} + +# +# Synopsis: display_remote_versions version +# + +function display_remote_versions() { + local version="$1" + update_mirror_settings_for_version "${version}" + local match='.' + local match_count="${N_MAX_REMOTE_MATCHES}" + + # Transform some labels before processing further. + if is_node_support_version "${version}"; then + version="$(display_latest_node_support_alias "${version}")" + match_count=1 + elif [[ "${version}" = "auto" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_auto_version || return 2 + version="${g_target_node}" + elif [[ "${version}" = "engine" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_engine_version || return 2 + version="${g_target_node}" + fi + + if [[ -z "${version}" ]]; then + match='.' + elif [[ "${version}" = "lts" || "${version}" = "stable" ]]; then + match_count=1 + # Codename is last field, first one with a name is newest lts + match="${TAB_CHAR}[a-zA-Z]+\$" + elif [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match_count=1 + match='.' + elif is_numeric_version "${version}"; then + version="v${version#v}" + # Avoid restriction message if exact version + is_exact_numeric_version "${version}" && match_count=1 + # Quote any dots in version so they are literal for expression + match="${version//\./\.}" + # Avoid 1.2 matching 1.23 + match="^${match}[^0-9]" + elif is_lts_codename "${version}"; then + # Capitalise (could alternatively make grep case insensitive) + codename="$(echo "${version:0:1}" | tr '[:lower:]' '[:upper:]')${version:1}" + # Codename is last field + match="${TAB_CHAR}${codename}\$" + elif is_download_folder "${version}"; then + match='.' + elif is_download_version "${version}"; then + version="${version#"${g_mirror_folder_name}"/}" + if [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match_count=1 + match='.' + else + version="v${version#v}" + match="${version//\./\.}" + match="^${match}" # prefix + if is_numeric_version "${version}"; then + # Exact numeric match + match="${match}[^0-9]" + fi + fi + else + abort "invalid version '$1'" + fi + display_match_limit "${match_count}" + + # Implementation notes: + # - using awk rather than head so do not close pipe early on curl + # - restrict search to compatible files as not always available, or not at same time + # - return status of curl command (i.e. PIPESTATUS[0]) + display_remote_index \ + | n_grep -E "$(display_compatible_file_field)" \ + | n_grep -E "${match}" \ + | awk "NR<=${match_count}" \ + | cut -f 1 \ + | n_grep -E -o '[^v].*' + return "${PIPESTATUS[0]}" +} + +# +# Synopsis: delete_with_echo target +# + +function delete_with_echo() { + if [[ -e "$1" ]]; then + echo "$1" + rm -rf "$1" + fi +} + +# +# Synopsis: uninstall_installed +# Uninstall the installed node and npm (leaving alone the cache), +# so undo install, and may expose possible system installed versions. +# + +uninstall_installed() { + # npm: https://docs.npmjs.com/misc/removing-npm + # rm -rf /usr/local/{lib/node{,/.npm,_modules},bin,share/man}/npm* + # node: https://stackabuse.com/how-to-uninstall-node-js-from-mac-osx/ + # Doing it by hand rather than scanning cache, so still works if cache deleted first. + # This covers tarballs for at least node 4 through 10. + + while true; do + read -r -p "Do you wish to delete node and npm from ${N_PREFIX}? " yn + case $yn in + [Yy]* ) break ;; + [Nn]* ) exit ;; + * ) echo "Please answer yes or no.";; + esac + done + + echo "" + echo "Uninstalling node and npm" + delete_with_echo "${N_PREFIX}/bin/node" + delete_with_echo "${N_PREFIX}/bin/npm" + delete_with_echo "${N_PREFIX}/bin/npx" + delete_with_echo "${N_PREFIX}/bin/corepack" + delete_with_echo "${N_PREFIX}/include/node" + delete_with_echo "${N_PREFIX}/lib/dtrace/node.d" + delete_with_echo "${N_PREFIX}/lib/node_modules/npm" + delete_with_echo "${N_PREFIX}/lib/node_modules/corepack" + delete_with_echo "${N_PREFIX}/share/doc/node" + delete_with_echo "${N_PREFIX}/share/man/man1/node.1" + delete_with_echo "${N_PREFIX}/share/systemtap/tapset/node.stp" +} + +# +# Synopsis: show_permission_suggestions +# + +function show_permission_suggestions() { + echo "Suggestions:" + echo "- run n with sudo, or" + if [[ "${N_CACHE_PREFIX}" == "${N_PREFIX}" ]]; then + echo "- define N_PREFIX to a writeable location, or" + else + echo "- define N_PREFIX and N_CACHE_PREFIX to writeable locations, or" + fi +} + +# +# Synopsis: show_diagnostics +# Show environment and check for common problems. +# + +function show_diagnostics() { + echo "This information is to help you diagnose issues, and useful when reporting an issue." + echo "Note: some output may contain passwords. Redact before sharing." + + printf "\n\nCOMMAND LOCATIONS AND VERSIONS\n" + + printf "\nbash\n" + command -v bash && bash --version + + printf "\nn\n" + command -v n && n --version + + printf "\nnode\n" + if command -v node &> /dev/null; then + command -v node && node --version + node -e 'if (process.versions.v8) console.log("JavaScript engine: v8");' + + printf "\nnpm\n" + command -v npm && npm --version + fi + + printf "\ntar\n" + if command -v tar &> /dev/null; then + command -v tar && tar --version + else + echo_red "tar not found. Needed for extracting downloads." + fi + + printf "\ncurl or wget\n" + if command -v curl &> /dev/null; then + command -v curl && curl --version + elif command -v wget &> /dev/null; then + command -v wget && wget --version + else + echo_red "Neither curl nor wget found. Need one of them for downloads." + fi + + printf "\nuname\n" + uname -a + + printf "\n\nSETTINGS\n" + + printf "\nn\n" + echo "node mirror: ${N_NODE_MIRROR}" + echo "node downloads mirror: ${N_NODE_DOWNLOAD_MIRROR}" + echo "install destination: ${N_PREFIX}" + [[ -n "${N_PREFIX}" ]] && echo "PATH: ${PATH}" + echo "ls-remote max matches: ${N_MAX_REMOTE_MATCHES}" + [[ -n "${N_PRESERVE_NPM}" ]] && echo "installs preserve npm by default" + [[ -n "${N_PRESERVE_COREPACK}" ]] && echo "installs preserve corepack by default" + + printf "\nProxy\n" + # disable "var is referenced but not assigned": https://github.com/koalaman/shellcheck/wiki/SC2154 + # shellcheck disable=SC2154 + [[ -n "${http_proxy}" ]] && echo "http_proxy: ${http_proxy}" + # shellcheck disable=SC2154 + [[ -n "${https_proxy}" ]] && echo "https_proxy: ${https_proxy}" + if command -v curl &> /dev/null; then + # curl supports lower case and upper case! + # shellcheck disable=SC2154 + [[ -n "${all_proxy}" ]] && echo "all_proxy: ${all_proxy}" + [[ -n "${ALL_PROXY}" ]] && echo "ALL_PROXY: ${ALL_PROXY}" + [[ -n "${HTTP_PROXY}" ]] && echo "HTTP_PROXY: ${HTTP_PROXY}" + [[ -n "${HTTPS_PROXY}" ]] && echo "HTTPS_PROXY: ${HTTPS_PROXY}" + if [[ -e "${CURL_HOME}/.curlrc" ]]; then + echo "have \$CURL_HOME/.curlrc" + elif [[ -e "${HOME}/.curlrc" ]]; then + echo "have \$HOME/.curlrc" + fi + elif command -v wget &> /dev/null; then + if [[ -e "${WGETRC}" ]]; then + echo "have \$WGETRC" + elif [[ -e "${HOME}/.wgetrc" ]]; then + echo "have \$HOME/.wgetrc" + fi + fi + + printf "\n\nCHECKS\n" + + printf "\nChecking n install destination is in PATH...\n" + local install_bin="${N_PREFIX}/bin" + local path_wth_guards=":${PATH}:" + if [[ "${path_wth_guards}" =~ :${install_bin}/?: ]]; then + printf "good\n" + else + echo_red "'${install_bin}' is not in PATH" + fi + if command -v node &> /dev/null; then + printf "\nChecking n install destination priority in PATH...\n" + local node_dir="$(dirname "$(command -v node)")" + + local index=0 + local path_entry + local path_entries + local install_bin_index=0 + local node_index=999 + IFS=':' read -ra path_entries <<< "${PATH}" + for path_entry in "${path_entries[@]}"; do + (( index++ )) + [[ "${path_entry}" =~ ^${node_dir}/?$ ]] && node_index="${index}" + [[ "${path_entry}" =~ ^${install_bin}/?$ ]] && install_bin_index="${index}" + done + if [[ "${node_index}" -lt "${install_bin_index}" ]]; then + echo_red "There is a version of node installed which will be found in PATH before the n installed version." + else + printf "good\n" + fi + fi + + # Check npm too. Simpler check than for PATH and node, more like the runtime logging for active/installed node. + if [[ -z "${N_PRESERVE_NPM}" ]]; then + printf "\nChecking npm install destination...\n" + local installed_npm="${N_PREFIX}/bin/npm" + local active_npm="$(command -v npm)" + if [[ -e "${active_npm}" && -e "${installed_npm}" && "${active_npm}" != "${installed_npm}" ]]; then + echo_red "There is an active version of npm shadowing the version installed by n. Check order of entries in PATH." + log "installed" "${installed_npm}" + log "active" "${active_npm}" + else + printf "good\n" + fi + fi + + printf "\nChecking prefix folders...\n" + if [[ ! -e "${N_PREFIX}" ]]; then + echo "Folder does not exist: ${N_PREFIX}" + echo "- This folder will be created when you do an install." + fi + if [[ "${N_PREFIX}" != "${N_CACHE_PREFIX}" && ! -e "${N_CACHE_PREFIX}" ]]; then + echo "Folder does not exist: ${N_CACHE_PREFIX}" + echo "- This folder will be created when you do an install." + fi + if [[ -e "${N_PREFIX}" && -e "${N_CACHE_PREFIX}" ]]; then + echo "good" + fi + + if [[ -e "${N_CACHE_PREFIX}" ]]; then + printf "\nChecking permissions for cache folder...\n" + # Using knowledge cache path ends in /n/versions in following check. + if [[ ! -e "${CACHE_DIR}" && (( -e "${N_CACHE_PREFIX}/n" && ! -w "${N_CACHE_PREFIX}/n" ) || ( ! -e "${N_CACHE_PREFIX}/n" && ! -w "${N_CACHE_PREFIX}" )) ]]; then + echo_red "You do not have write permission to create: ${CACHE_DIR}" + show_permission_suggestions + echo "- make a folder you own:" + echo " sudo mkdir -p \"${CACHE_DIR}\"" + echo " sudo chown $(whoami) \"${CACHE_DIR}\"" + elif [[ ! -e "${CACHE_DIR}" ]]; then + echo "Cache folder does not exist: ${CACHE_DIR}" + echo "- This is normal if you have not done an install yet, as cache is only created when needed." + elif [[ ! -w "${CACHE_DIR}" ]]; then + echo_red "You do not have write permission to: ${CACHE_DIR}" + show_permission_suggestions + echo "- change folder ownership to yourself:" + echo " sudo chown -R $(whoami) \"${CACHE_DIR}\"" + else + echo "good" + fi + fi + + if [[ -e "${N_PREFIX}" ]]; then + printf "\nChecking permissions for install folders...\n" + local install_writeable="true" + for subdir in bin lib include share; do + if [[ -e "${N_PREFIX}/${subdir}" && ! -w "${N_PREFIX}/${subdir}" ]]; then + install_writeable="false" + echo_red "You do not have write permission to: ${N_PREFIX}/${subdir}" + break + fi + if [[ ! -e "${N_PREFIX}/${subdir}" && ! -w "${N_PREFIX}" ]]; then + install_writeable="false" + echo_red "You do not have write permission to create: ${N_PREFIX}/${subdir}" + break + fi + done + if [[ "${install_writeable}" = "true" ]]; then + echo "good" + else + show_permission_suggestions + echo "- change folder ownerships to yourself:" + echo " cd \"${N_PREFIX}\"" + echo " sudo mkdir -p bin lib include share" + echo " sudo chown -R $(whoami) bin lib include share" + fi + fi + + printf "\nChecking mirror is reachable...\n" + if is_ok "${N_NODE_MIRROR}/"; then + printf "good\n" + else + echo_red "mirror not reachable" + printf "Showing failing command and output\n" + if command -v curl &> /dev/null; then + ( set -x; do_get --head "${N_NODE_MIRROR}/" ) + else + ( set -x; do_get --spider "${N_NODE_MIRROR}/" ) + printf "\n" + fi + fi +} + +# +# Handle arguments. +# + +# First pass. Process the options so they can come before or after commands, +# particularly for `n lsr --all` and `n install --arch x686` +# which feel pretty natural. + +unprocessed_args=() +positional_arg="false" + +while [[ $# -ne 0 ]]; do + case "$1" in + --all) N_MAX_REMOTE_MATCHES=32000 ;; + -V|--version) display_n_version ;; + -h|--help|help) display_help; exit ;; + -q|--quiet) set_quiet ;; + -d|--download) DOWNLOAD="true" ;; + --offline) OFFLINE="true" ;; + --insecure) set_insecure ;; + -p|--preserve) N_PRESERVE_NPM="true" N_PRESERVE_COREPACK="true" ;; + --no-preserve) N_PRESERVE_NPM="" N_PRESERVE_COREPACK="" ;; + --use-xz) N_USE_XZ="true" ;; + --no-use-xz) N_USE_XZ="false" ;; + --latest) display_remote_versions latest; exit ;; + --stable) display_remote_versions lts; exit ;; # [sic] old terminology + --lts) display_remote_versions lts; exit ;; + -a|--arch) shift; set_arch "$1";; # set arch and continue + exec|run|as|use) + unprocessed_args+=( "$1" ) + positional_arg="true" + ;; + *) + if [[ "${positional_arg}" == "true" ]]; then + unprocessed_args+=( "$@" ) + break + fi + unprocessed_args+=( "$1" ) + ;; + esac + shift +done + +if [[ -z "${N_USE_XZ+defined}" ]]; then + N_USE_XZ="true" # Default to using xz + can_use_xz || N_USE_XZ="false" +fi + +set -- "${unprocessed_args[@]}" + +if test $# -eq 0; then + test -z "$(display_versions_paths)" && err_no_installed_print_help + menu_select_cache_versions +else + while test $# -ne 0; do + case "$1" in + bin|which) display_bin_path_for_version "$2"; exit ;; + run|as|use) shift; run_with_version "$@"; exit ;; + exec) shift; exec_with_version "$@"; exit ;; + doctor) show_diagnostics; exit ;; + rm|-) shift; remove_versions "$@"; exit ;; + prune) prune_cache; exit ;; + latest) install latest; exit ;; + stable) install stable; exit ;; + lts) install lts; exit ;; + ls|list) display_versions_paths; exit ;; + lsr|ls-remote|list-remote) shift; display_remote_versions "$1"; exit ;; + uninstall) uninstall_installed; exit ;; + i|install) shift; install "$1"; exit ;; + N_TEST_DISPLAY_LATEST_RESOLVED_VERSION) shift; get_latest_resolved_version "$1" > /dev/null || exit 2; echo "${g_target_node}"; exit ;; + *) install "$1"; exit ;; + esac + shift + done +fi