diff --git a/helpers/helpers.v2.d/apps b/helpers/helpers.v2.d/apps new file mode 100644 index 000000000..8f80fdfe3 --- /dev/null +++ b/helpers/helpers.v2.d/apps @@ -0,0 +1,169 @@ +#!/bin/bash + +ynh::apps::list() { + yunohost app list --output-as json --quiet | jq -r '.apps[].id' +} + + +# Install others YunoHost apps +# +# usage: ynh_install_apps --apps="appfoo?domain=domain.foo&path=/foo appbar?domain=domain.bar&path=/bar&admin=USER&language=fr&is_public=1&pass?word=pass&port=666" +# | arg: -a, --apps= - apps to install +# +# Requires YunoHost version *.*.* or higher. +ynh::apps::install() { + local -a apps=() + arguments::parse \ + "-a|--apps)apps;Array,R" \ + -- "${@+$@}" + + local -a installed_apps + mapfile -t installed_apps < <(ynh::apps::list) + + for app_and_args in "${apps[@]}"; do + local app_name app_args + app_name="$(cut -d "?" -f1 <<< "$app_and_args")" + app_args="$(cut -d "?" -f2- -s <<< "$app_and_args")" + if string::empty "$app_name"; then + log::panic "You didn't provided a YunoHost app to install" + fi + + yunohost tools update apps + + if list::contains "$app_name" "${installed_apps[@]}"; then + # App already installed, upgrade it + yunohost app upgrade "$app_name" + else + # Install the app with its arguments + yunohost app install "$app_name" "$app_args" + fi + done + + ynh::setting::set --key "apps_dependencies" --value "$(array::join ", " "${apps[@]}")" +} + +# Remove other YunoHost apps +# +# Other YunoHost apps will be removed only if no other apps need them. +# +# usage: ynh_remove_apps +# +# Requires YunoHost version *.*.* or higher. +ynh::apps::remove_deps() { + local -a to_remove + string::split_to to_remove ", " "$(ynh::setting::get --key "apps_dependencies")" + ynh::setting::delete --key "apps_dependencies" + + # Make the array of reverse dependencies + local -A reverse_deps + for app in $(ynh::apps::list); do + local -a app_deps + string::split_to app_deps ", " "$(ynh::setting::get --app="$app" --key=apps_dependencies)" + for dep in "${app_deps[@]}"; do + reverse_deps[$dep]="${reverse_deps[$dep]}, $app" + done + done + + # Now remove all deps that have no other reverse dependencies + for app in "${to_remove[@]}"; do + if string::empty "${reverse_deps[$app]}"; then + log::info "Removing $app..." + yunohost app remove "$app" --purge + else + log::info "$app was not removed because it's still required by ${reverse_deps[$app]}." + fi + done +} + +# Spawn a Bash shell with the app environment loaded +# +# usage: ynh_spawn_app_shell --app="app" +# | arg: -a, --app= - the app ID +# +# examples: +# ynh_spawn_app_shell --app="APP" <<< 'echo "$USER"' +# ynh_spawn_app_shell --app="APP" < /tmp/some_script.bash +# +# Requires YunoHost version 11.0.* or higher, and that the app relies on packaging v2 or higher. +# The spawned shell will have environment variables loaded and environment files sourced +# from the app's service configuration file (defaults to $app.service, overridable by the packager with `service` setting). +# If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings. +ynh::apps::shell_into() { + local app + arguments::parse \ + "-a|--app)apps;String,R" \ + -- "${@+$@}" + + + # Force Bash to be used to run this helper + if [[ ! "$0" =~ \/?bash$ ]]; then + log::panic "Please use Bash as shell." + fi + + # Make sure the app is installed + local installed_apps + mapfile -t installed_apps < <(ynh::apps::list) + if ! array::contains "$app" installed_apps; then + log::panic "$app is not in the apps list" + fi + + # Make sure the app has its own user + if ! id -u "$app" &>/dev/null; then + log::panic "There is no '$app' system user" + fi + + # Make sure the app has an install_dir setting + local install_dir + install_dir="$(ynh::setting::get --app "$app" --key install_dir)" + if string::empty "$install_dir"; then + log::panic "$app has no install_dir setting (does it use packaging format >=2?)" + fi + + # Load the app's service name, or default to $app + local service + service="$(ynh::setting::get --app "$app" --key service)" + if string::empty "$service"; then + service="$app" + fi + + # Load the service variables + local -a env_vars + string::split_to env_vars " " "$(systemctl show "$service.service" -p "Environment" --value)" + + local -a env_files + string::split_to env_files " " "$(systemctl show "$service.service" -p "EnvironmentFiles" --value)" + + local env_dir + env_dir="$(systemctl show "$service.service" -p "WorkingDirectory" --value)" + + # Force `php` to its intended version + # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export` + local phpversion phpflags + phpversion="$(ynh::setting::get --app "$app" --key phpversion)" + phpflags="$(ynh::setting::get --app "$app" --key phpflags)" + if ! string::empty "$phpversion"; then + phpstring="php() { php${phpversion} ${phpflags} \"\$@\"; }" + fi + + ( + export HOME="$install_dir" + for var in "${env_vars[@]}"; do + export "${var?}" + done + for file in "${env_files[@]}"; do + set -a + source "$file" + set +a + done + + if ! string::empty "$phpstring"; then + eval "${phpstring:-}" + export -f php + fi + + if ! string::empty "$env_dir"; then + cd "$env_dir" + fi + su -s /bin/bash "$app" + ) +} diff --git a/helpers/helpers.v2.d/bash-modules/arguments.sh b/helpers/helpers.v2.d/bash-modules/arguments.sh new file mode 100644 index 000000000..e8eb72aab --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/arguments.sh @@ -0,0 +1,301 @@ +#!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> `arguments` - contains function to parse arguments and assign option values to variables. + +#>> +#>> ## FUNCTIONS + +#>> +#>> * `arguments::parse [-S|--FULL)VARIABLE;FLAGS[,COND]...]... -- [ARGUMENTS]...` +#> +#> Where: +#> +#> * `-S` - short option name. +#> +#> * `--FULL` - long option name. +#> +#> * `VARIABLE` - name of shell variable to assign value to. +#> +#> * `FLAGS` - one of (case sensitive): +#> * `Y | Yes` - set variable value to "yes"; +#> * `No` - set variable value to "no"; +#> * `I | Inc | Incremental` - incremental (no value) - increment variable value by one; +#> * `S | Str | String` - string value; +#> * `N | Num | Number` - numeric value; +#> * `A | Arr | Array` - array of string values (multiple options); +#> * `C | Com | Command` - option name will be assigned to the variable. +#> +#> * `COND` - post conditions: +#> * `R | Req | Required` - option value must be not empty after end of parsing. +#> Set initial value to empty value to require this option; +#> * any code - executed after option parsing to check post conditions, e.g. "(( FOO > 3 )), (( FOO > BAR ))". +#> +#> * -- - the separator between option descriptions and script commandline arguments. +#> +#> * `ARGUMENTS` - command line arguments to parse. +#> +#> **LIMITATION:** grouping of one-letter options is NOT supported. Argument `-abc` will be parsed as +#> option `-abc`, not as `-a -b -c`. +#> +#> **NOTE:** bash4 requires to use `"${@:+$@}"` to expand empty list of arguments in strict mode (`-u`). +#> +#> By default, function supports `-h|--help`, `--man` and `--debug` options. +#> Options `--help` and `--man` are calling `arguments::help()` function with `2` or `1` as +#> argument. Override that function if you want to provide your own help. +#> +#> Unlike many other parsers, this function stops option parsing at first +#> non-option argument. +#> +#> Use `--` in commandline arguments to strictly separate options and arguments. +#> +#> After option parsing, unparsed command line arguments are stored in +#> `ARGUMENTS` array. +#> +#> **Example:** +#> +#> ```bash +#> # Boolean variable ("yes" or "no") +#> FOO="no" +#> # String variable +#> BAR="" +#> # Indexed array +#> declare -a BAZ=( ) +#> # Integer variable +#> declare -i TIMES=0 +#> +#> arguments::parse \\ +#> "-f|--foo)FOO;Yes" \\ +#> "-b|--bar)BAR;String,Required" \\ +#> "-B|--baz)BAZ;Array" \\ +#> "-i|--inc)TIMES;Incremental,((TIMES<3))" \\ +#> -- \\ +#> "${@:+$@}" +#> +#> # Print name and value of variables +#> dbg FOO BAR BAZ TIMES ARGUMENTS +#> ``` +arguments::parse() { + + # Global array to hold command line arguments + ARGUMENTS=( ) + + # Local variables + local OPTION_DESCRIPTIONS PARSER + declare -a OPTION_DESCRIPTIONS + # Initialize array, because declare -a is not enough anymore for -u opt + OPTION_DESCRIPTIONS=( ) + + # Split arguments list at "--" + while [ $# -gt 0 ] + do + [ "$1" != "--" ] || { + shift + break + } + + OPTION_DESCRIPTIONS[${#OPTION_DESCRIPTIONS[@]}]="$1" # Append argument to end of array + shift + done + + # Generate option parser and execute it + PARSER="$(arguments::generate_parser "${OPTION_DESCRIPTIONS[@]:+${OPTION_DESCRIPTIONS[@]}}")" || return 1 + eval "$PARSER" || return 1 + arguments::parse_options "${@:+$@}" || return $? +} + +#>> +#>> * `arguments::generate_parser OPTIONS_DESCRIPTIONS` - generate parser for options. +#> Will create function `arguments::parse_options()`, which can be used to parse arguments. +#> Use `declare -fp arguments::parse_options` to show generated source. +arguments::generate_parser() { + + local OPTION_DESCRIPTION OPTION_CASE OPTION_FLAGS OPTION_TYPE OPTION_OPTIONS OPTIONS_PARSER="" OPTION_POSTCONDITIONS="" + + # Parse option description and generate code to parse that option from script arguments + while [ $# -gt 0 ] + do + # Parse option description + OPTION_DESCRIPTION="$1" ; shift + + # Check option syntax + case "$OPTION_DESCRIPTION" in + *')'*';'*) ;; # OK + *) + log::error "Incorrect syntax of option: \"$OPTION_DESCRIPTION\". Option syntax: -S|--FULL)VARIABLE;TYPE[,CHECK]... . Example: '-f|--foo)FOO;String,Required'." + __log__DEBUG=yes; log::stacktrace + return 1 + ;; + esac + + OPTION_CASE="${OPTION_DESCRIPTION%%)*}" # Strip everything after first ')': --foo)BAR -> --foo + OPTION_VARIABLE="${OPTION_DESCRIPTION#*)}" # Strip everything before first ')': --foo)BAR -> BAR + OPTION_FLAGS="${OPTION_VARIABLE#*;}" # Strip everything before first ';': BAR;Baz -> Baz + OPTION_VARIABLE="${OPTION_VARIABLE%%;*}" # String everything after first ';': BAR;Baz -> BAR + + IFS=',' read -a OPTION_OPTIONS <<<"$OPTION_FLAGS" # Convert string into array: 'a,b,c' -> [ a b c ] + OPTION_TYPE="${OPTION_OPTIONS[0]:-}" ; unset OPTION_OPTIONS[0] ; # First element of array is option type + + # Generate the parser for option + case "$OPTION_TYPE" in + + Y|Yes) # Set variable to "yes", no arguments + OPTIONS_PARSER="$OPTIONS_PARSER + $OPTION_CASE) + $OPTION_VARIABLE=\"yes\" + shift 1 + ;; + " + ;; + + No) # Set variable to "no", no arguments + OPTIONS_PARSER="$OPTIONS_PARSER + $OPTION_CASE) + $OPTION_VARIABLE=\"no\" + shift 1 + ;; + " + ;; + + C|Com|Command) # Set variable to name of the option, no arguments + OPTIONS_PARSER="$OPTIONS_PARSER + $OPTION_CASE) + $OPTION_VARIABLE=\"\$1\" + shift 1 + ;; + " + ;; + + + I|Incr|Incremental) # Incremental - any use of this option will increment variable by 1 + OPTIONS_PARSER="$OPTIONS_PARSER + $OPTION_CASE) + let $OPTION_VARIABLE++ || : + shift 1 + ;; + " + ;; + + S|Str|String) # Regular option with string value + OPTIONS_PARSER="$OPTIONS_PARSER + $OPTION_CASE) + $OPTION_VARIABLE=\"\${2:?ERROR: String value is required for \\\"$OPTION_CASE\\\" option. See --help for details.}\" + shift 2 + ;; + ${OPTION_CASE//|/=*|}=*) + $OPTION_VARIABLE=\"\${1#*=}\" + shift 1 + ;; + " + ;; + + N|Num|Number) # Same as string + OPTIONS_PARSER="$OPTIONS_PARSER + $OPTION_CASE) + $OPTION_VARIABLE=\"\${2:?ERROR: Numeric value is required for \\\"$OPTION_CASE\\\" option. See --help for details.}\" + shift 2 + ;; + ${OPTION_CASE//|/=*|}=*) + $OPTION_VARIABLE=\"\${1#*=}\" + shift 1 + ;; + " + ;; + + A|Arr|Array) # Array of strings + OPTIONS_PARSER="$OPTIONS_PARSER + $OPTION_CASE) + ${OPTION_VARIABLE}[\${#${OPTION_VARIABLE}[@]}]=\"\${2:?Value is required for \\\"$OPTION_CASE\\\". See --help for details.}\" + shift 2 + ;; + ${OPTION_CASE//|/=*|}=*) + ${OPTION_VARIABLE}[\${#${OPTION_VARIABLE}[@]}]=\"\${1#*=}\" + shift 1 + ;; + " + ;; + + *) + echo "ERROR: Unknown option type: \"$OPTION_TYPE\"." >&2 + return 1 + ;; + esac + + # Parse option options, e.g "Required". Any other text is treated as condition, e.g. (( VAR > 10 && VAR < 20 )) + local OPTION_OPTION + for OPTION_OPTION in "${OPTION_OPTIONS[@]:+${OPTION_OPTIONS[@]}}" + do + case "$OPTION_OPTION" in + R|Req|Required) + OPTION_POSTCONDITIONS="$OPTION_POSTCONDITIONS + [ -n \"\$${OPTION_VARIABLE}\" ] || { echo \"ERROR: Option \\\"$OPTION_CASE\\\" is required. See --help for details.\" >&2; return 1; } + " + ;; + *) # Any other code after option type i + OPTION_POSTCONDITIONS="$OPTION_POSTCONDITIONS + $OPTION_OPTION || { echo \"ERROR: Condition for \\\"$OPTION_CASE\\\" option is failed. See --help for details.\" >&2; return 1; } + " + ;; + esac + done + + done + echo " + arguments::parse_options() { + # Global array to hold command line arguments + ARGUMENTS=( ) + + while [ \$# -gt 0 ] + do + case \"\$1\" in + # User options. + $OPTIONS_PARSER + + # Built-in options. + -h|--help) + arguments::help 2 + exit 0 + ;; + --man) + arguments::help 1 + exit 0 + ;; + --debug) + log::enable_debug_mode + shift + ;; + --) + shift; break; # Do not parse rest of the command line arguments + ;; + -*) + echo \"ERROR: Unknown option: \\\"\$1\\\".\" >&2 + arguments::help 3 + return 1 + ;; + *) + break; # Do not parse rest of the command line + ;; + esac + done + [ \$# -eq 0 ] || ARGUMENTS=( \"\$@\" ) # Store rest of the command line arguments into the ARGUMENTS array + $OPTION_POSTCONDITIONS + } + " +} + +#>> +#>> * `arguments::help LEVEL` - display embeded documentation. +#>> LEVEL - level of documentation: +#>> * 3 - summary (`#>>>` comments), +#>> * 2 - summary and usage ( + `#>>` comments), +#>> * 1 - full documentation (+ `#>` comments). +arguments::help() { + local LEVEL="${1:-3}" + case "$LEVEL" in + 2|3) import::show_documentation "$LEVEL" "$IMPORT__BIN_FILE" ;; + 1) import::show_documentation "$LEVEL" "$IMPORT__BIN_FILE" | less ;; + esac +} diff --git a/helpers/helpers.v2.d/bash-modules/array.sh b/helpers/helpers.v2.d/bash-modules/array.sh new file mode 100644 index 000000000..8d7cc9769 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/array.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> list - various functions to manipulate lists (passed as argument list). +#>>> array - various functions to manipulate arrays (passed as array names). + +#>> +#>> ## FUNCTIONS + +list::contain() { + local searching="$1"; shift + if [[ -z "$searching" ]] || [[ "$#" = 0 ]]; then + return 1 + fi + for element in "$@"; do + if [[ "$searching" == "$element" ]]; then + return 0 + fi + done + return 1 +} + +array::contains() { + local searching="$1"; shift + declare -n __array_name="$1" ; shift + list::contain "$searching" "${__array_name[@]}" +} + + +list::join() { + local sep="$1"; shift + while (( "$#" > 1 )); do + printf '%s%s' "$1" "$sep" + shift + done + if [[ "$#" = 1 ]]; then + printf '%s' "$1" + fi +} + +array::join() { + local sep="$1"; shift + declare -n __array_name="$1"; shift + list::join "$sep" "${__array_name[@]}" +} + + +list::min() { + local __min + if [[ "$#" = 0 ]]; then + return 1 + fi + for value in "$@"; do + __min=$(echo "if($value<$__min) $value else $__min" | bc) + done + echo "$__min" +} + +array::min() { + local sep="$1"; shift + declare -n __array_name="$1"; shift + list::min "$sep" "${__array_name[@]}" +} + + +list::max() { + local __max + if [[ "$#" = 0 ]]; then + return 1 + fi + for value in "$@"; do + __max=$(echo "if($value<$__max) $value else $__max" | bc) + done + echo "$__max" +} + +array::max() { + local sep="$1"; shift + declare -n __array_name="$1"; shift + list::max "$sep" "${__array_name[@]}" +} + + +array::sort() { + declare -n __array_name="$1" ; shift + local __sort_args=("$@") + + printf '%s\n' "${__array_name[@]}" | sort "${__sort_args[@]}" +} + +list::sort() { + local __array=("$@") + array::sort __array +} + + +list::uniq() { + printf "%s\n" "$@" | sort -u +} + +array::uniq() { + local sep="$1"; shift + declare -n __array_name="$1"; shift + list::uniq "$sep" "${__array_name[@]}" +} + +array::empty() { + declare -n __array_name="$1"; shift + (( ${#__array_name[@]} == 0 )) +} diff --git a/helpers/helpers.v2.d/bash-modules/cd_to_bindir.sh b/helpers/helpers.v2.d/bash-modules/cd_to_bindir.sh new file mode 100644 index 000000000..5fffdf1cf --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/cd_to_bindir.sh @@ -0,0 +1,33 @@ +##!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ +#>> # NAME +#>>> `cd_to_bindir` - change working directory to the directory where main script file is located. +#>> +#>> # DESCRIPTION +#>> +#>> Just import this cwdir module to change current working directory to a directory, +#>> where main script file is located. + +# Get file name of the main source file +__CD_TO_BINDIR__BIN_FILE="${BASH_SOURCE[${#BASH_SOURCE[@]}-1]}" + +# If file name doesn't contains "/", then use `which` to find path to file name. +[[ "$__CD_TO_BINDIR__BIN_FILE" == */* ]] || __CD_TO_BINDIR__BIN_FILE=$( which "$__CD_TO_BINDIR__BIN_FILE" ) + +# Strip everything after last "/" to get directoru: "/foo/bar/baz" -> "/foo/bar", "./foo" -> "./". +# Then cd to the directory and get it path. +__CD_TO_BINDIR_DIRECTORY=$( cd "${__CD_TO_BINDIR__BIN_FILE%/*}/" ; pwd ) + +unset __CD_TO_BINDIR__BIN_FILE + +#>> +#>> # FUNCTIONS +#>> +#>> * `ch_bin_dir` - Change working directory to directory where script is located, which is usually called "bin dir". +cd_to_bindir() { + cd "$__CD_TO_BINDIR_DIRECTORY" +} + +# Call this function at import. +cd_to_bindir diff --git a/helpers/helpers.v2.d/bash-modules/date.sh b/helpers/helpers.v2.d/bash-modules/date.sh new file mode 100644 index 000000000..7c5b370b9 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/date.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> `date` - date-time functions. + +#> +#> ## FUNCTIONS + +#>> +#>> * `date::timestamp VARIABLE` - return current time in seconds since UNIX epoch +date::timestamp() { + printf -v "$1" "%(%s)T" "-1" +} + +#>> +#>> * `date::current_datetime VARIABLE FORMAT` - return current date time in given format. +#> See `man 3 strftime` for details. +date::current_datetime() { + printf -v "$1" "%($2)T" "-1" +} + +#>> +#>> * `date::print_current_datetime FORMAT` - print current date time in given format. +#> See `man 3 strftime` for details. +date::print_current_datetime() { + printf "%($1)T" "-1" +} + +#>> +#>> * `date::datetime VARIABLE FORMAT TIMESTAMP` - return current date time in given format. +#> See `man 3 strftime` for details. +date::datetime() { + printf -v "$1" "%($2)T" "$3" +} + +#>> +#>> * `date::print_elapsed_time` - print value of SECONDS variable in human readable form: "Elapsed time: 0 days 00:00:00." +#> It's useful to know time of execution of a long script, so here is function for that. +#> Assign 0 to SECONDS variable to reset counter. +date::print_elapsed_time() { + printf "Elapsed time: %d days %02d:%02d:%02d.\n" $((SECONDS/(24*60*60))) $(((SECONDS/(60*60))%24)) $(((SECONDS/60)%60)) $((SECONDS%60)) +} diff --git a/helpers/helpers.v2.d/bash-modules/import.sh b/helpers/helpers.v2.d/bash-modules/import.sh new file mode 100755 index 000000000..372b7653a --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/import.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2009-2013 Volodymyr M. Lisivka , All Rights Reserved +# +# This file is part of bash-modules (http://trac.assembla.com/bash-modules/). +# +# bash-modules is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 2.1 of the License, or +# (at your option) any later version. +# +# bash-modules is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with bash-modules If not, see . + +#> ## NAME +#>> +#>>> `import.sh` - import bash modules into scripts or into interactive shell +#> +#> ## SYNOPSIS +#>> +#>> ### In a scipt: +#>> +#>> * `. import.sh MODULE[...]` - import module(s) into script or shell +#>> * `source import.sh MODULE[...]` - same as above, but with `source` instead of `.` +#>> +#>> +#>> ### At command line: +#>> +#>> * `import.sh --help|-h` - print this help text +#>> * `import.sh --man` - show manual +#>> * `import.sh --list` - list modules with their path +#>> * `import.sh --summary|-s [MODULE...]` - list module(s) with summary +#>> * `import.sh --usage|-u MODULE[...]` - print module help text +#>> * `import.sh --doc|-d MODULE[...]` - print module documentation +#>> +#> ## DESCRIPTION +#> +#> Imports given module(s) into current shell. +#> +#> Use: +#> +#> * `import.sh --list` - to print list of available modules. +#> * `import.sh --summary` - to print list of available modules with short description. +#> * `import.sh --usage MODULE[...]` - to print longer description of given module(s). + +[ "${__IMPORT__DEFINED:-}" == "yes" ] || { + __IMPORT__DEFINED="yes" + + [ "$BASH_VERSINFO" -ge 4 ] || { + echo "[import.sh] ERROR: This script works only with Bash, version 4 or greater. Upgrade is necessary." >&2 + exit 80 + } + + # If BASH_MODULES_PATH variable contains a ':' separator, then split it into array + if [[ "${BASH_MODULES_PATH:-}" == *':'* ]]; then + __split_by_delimiter() { + local __string__VAR="$1" + local IFS="$2" + local __string__VALUE="${3:-}" + read -a "$__string__VAR" <<<"${__string__VALUE:-}" + } + __split_by_delimiter __BASH_MODULES_PATH_ARRAY ':' "${BASH_MODULES_PATH:+$BASH_MODULES_PATH}" + unset -f __split_by_delimiter + else + __BASH_MODULES_PATH_ARRAY=( "${BASH_MODULES_PATH:+$BASH_MODULES_PATH}" ) + fi + + #> + #> ## CONFIGURATION + + #> + #> * `BASH_MODULES_PATH` - (variable with single path entry, at present time). + #> `BASH_MODULES_PATH` can contain multiple directories separated by ":". + #> + #> * `__IMPORT__BASE_PATH` - array with list of your own directories with modules, + #> which will be prepended to module search path. You can set `__IMPORT__BASE_PATH` array in + #> script at begining, in `/etc/bash-modules/config.sh`, or in `~/.config/bash-modules/config.sh` file. + __IMPORT__BASE_PATH=( "${__BASH_MODULES_PATH_ARRAY[@]:+${__BASH_MODULES_PATH_ARRAY[@]}}" "${__IMPORT__BASE_PATH[@]:+${__IMPORT__BASE_PATH[@]}}" "/usr/share/bash-modules" ) + unset __BASH_MODULES_PATH_ARRAY + + #> + #> * `/etc/bash-modules/config.sh` - system wide configuration file. + #> WARNING: Code in this script will affect all scripts. + #> + #> ### Example configration file + #> + #> Put following snippet into `~/.config/bash-modules/config.sh` file: + #> + #>```bash + #> + #> # Enable stack trace printing for warnings and errors, + #> # like with --debug option: + #> __log__STACKTRACE=="yes" + #> + #> # Add additional directory to module search path: + #> BASH_MODULES_PATH="/home/user/my-bash-modules" + #> + #>``` + [ ! -s /etc/bash-modules/config.sh ] || source /etc/bash-modules/config.sh || { + echo "[import.sh] WARN: Cannot import \"/etc/bash-modules/config.sh\" or an error in this file." >&2 + } + + #> + #> * `~/.config/bash-modules/config.sh` - user configuration file. + #> **WARNING:** Code in this script will affect all user scripts. + [ ! -s "$HOME/.config/bash-modules/config.sh" ] || source "$HOME/.config/bash-modules/config.sh" || { + echo "[import.sh] WARN: Cannot import \"$HOME/.config/bash-modules/config.sh\" or an error in this file." >&2 + } + + #> + #> ## VARIABLES + + #> + #> * `IMPORT__BIN_FILE` - script main file name, e.g. `/usr/bin/my-script`, as in `$0` variable in main file. + __IMPORT_INDEX="${#BASH_SOURCE[*]}" + IMPORT__BIN_FILE="${BASH_SOURCE[__IMPORT_INDEX-1]}" + unset __IMPORT_INDEX + + #> + #> ## FUNCTIONS + + #> + #> * `import::import_module MODULE` - import single module only. + import::import_module() { + local __MODULE="${1:?Argument is required: module name, without path and without .sh extension, e.g. "log".}" + + local __PATH + for __PATH in "${__IMPORT__BASE_PATH[@]}" + do + [ -f "$__PATH/$__MODULE.sh" ] || continue + + # Don't import module twice, to avoid loops. + # Replace some special symbols in the module name by "_". + local -n __IMPORT_MODULE_DEFINED="__${__MODULE//[\[\]\{\}\/!@#$%^&*()=+~\`\\,?|\'\"-]/_}__DEFINED" # Variable reference + [ "${__IMPORT_MODULE_DEFINED:-}" != "yes" ] || return 0 # Already imported + __IMPORT_MODULE_DEFINED="yes" + unset -n __IMPORT_MODULE_DEFINED # Unset reference + + # Import module + source "$__PATH/$__MODULE.sh" || return 1 + return 0 + done + + echo "[import.sh:import_module] ERROR: Cannot locate module: \"$__MODULE\". Search path: ${__IMPORT__BASE_PATH[*]}" >&2 + return 2 + } + + #> + #> * `import::import_modules MODULE[...]` - import module(s). + import::import_modules() { + local __MODULE __ERROR_CODE=0 + for __MODULE in "$@" + do + import::import_module "$__MODULE" || __ERROR_CODE=$? + done + return $__ERROR_CODE + } + + #> + #> * `import::list_modules FUNC [MODULE]...` - print various information about module(s). + #> `FUNC` is a function to call on each module. Function will be called with two arguments: + #> path to module and module name. + #> Rest of arguments are module names. No arguments means all modules. + import::list_modules() { + local __FUNC="${1:?ERROR: Argument is required: function to call with module name.}" + shift + + declare -a __MODULES + local __PATH __MODULE __MODULES + + # Collect modules + if [ $# -eq 0 ] + then + # If no arguments are given, + # then add all modules in all directories + for __PATH in "${__IMPORT__BASE_PATH[@]}" + do + for __MODULE in "$__PATH"/*.sh + do + [ -f "$__MODULE" ] || continue + __MODULES[${#__MODULES[@]}]="$__MODULE" + done + done + else + # Argument can be directory or module path or module name. + local __ARG + for __ARG in "$@" + do + if [ -d "$__ARG" ] + then + # Directory. Add all modules in directory + for __MODULE in "$__ARG"/*.sh + do + [ -f "$__MODULE" ] || continue + __MODULES[${#__MODULES[@]}]="$__MODULE" + done + elif [ -f "$__ARG" ] + then + # Direct path. Add single module. + __MODULES[${#__MODULES[@]}]="$__ARG" + else + # Module name. Find single module in path. + for __PATH in "${__IMPORT__BASE_PATH[@]}" + do + [ -f "$__PATH/$__ARG.sh" ] || continue + __MODULES[${#__MODULES[@]}]="$__PATH/$__ARG.sh" + done + fi + done + fi + + # Call function on each module + local __MODULE_PATH + for __MODULE_PATH in "${__MODULES[@]}" + do + [ -f "$__MODULE_PATH" ] || continue + __MODULE="${__MODULE_PATH##*/}" # Strip directory + __MODULE="${__MODULE%.sh}" # Strip extension + + # Call requested function on each module + $__FUNC "$__MODULE_PATH" "$__MODULE" || { echo "WARN: Error in function \"$__FUNC '$__MODULE_PATH'\"." >&2 ; } + done + } + + #> + #> * `import::show_documentation LEVEL PARSER FILE` - print module built-in documentation. + #> This function scans given file for lines with "#>" prefix (or given prefix) and prints them to stdout with prefix stripped. + #> * `LEVEL` - documentation level (one line summary, usage, full manual): + #> - 1 - for manual (`#>` and `#>>` and `#>>>`), + #> - 2 - for usage (`#>>` and `#>>>`), + #> - 3 - for one line summary (`#>>>`), + #> - or arbitrary prefix, e.g. `##`. + #> * `FILE` - path to file with built-in documentation. + import::show_documentation() { + local LEVEL="${1:?ERROR: Argument is required: level of documentation: 1 for all documentation, 2 for usage, 3 for one line summary.}" + local FILE="${2:?ERRROR: Argument is required: file to parse documentation from.}" + + [ -e "$FILE" ] || { + echo "ERROR: File \"$FILE\" is not exits." >&2 + } + [ -f "$FILE" ] || { + echo "ERROR: Path \"$FILE\" is not a file." >&2 + } + [ -r "$FILE" ] || { + echo "ERROR: Cannot read file \"$FILE\"." >&2 + } + [ -s "$FILE" ] || { + echo "ERROR: File \"$FILE\" is empty." >&2 + } + + local PREFIX="" + case "$LEVEL" in + 1) PREFIX="#>" ;; + 2) PREFIX="#>>" ;; + 3) PREFIX="#>>>" ;; + *) + PREFIX="$LEVEL" + ;; + esac + + local line + while read line + do + if [[ "$line" =~ ^\s*"$PREFIX"\>*\ ?(.*)$ ]] + then + echo "${BASH_REMATCH[1]}" + fi + done < "$FILE" + } + + +} + +# If this is top level code and program name is .../import.sh +if [ "${FUNCNAME:+x}" == "" -a "${0##*/}" == "import.sh" ] +then + show_module_info() { + local module_path="$1" + local module_name="$2" + + printf "%-24s\t%s\n" "$module_name" "$module_path" + } + + # import.sh called as standalone program + if [ "$#" -eq 0 ] + then + import::show_documentation 2 "$IMPORT__BIN_FILE" + else + case "$1" in + --list|-l) + shift 1 + import::list_modules "show_module_info" "${@:+$@}" + ;; + --summary|-s) + shift 1 + import::list_modules "import::show_documentation 3" "${@:+$@}" + ;; + --usage|-u) + shift 1 + import::list_modules "import::show_documentation 2" "${@:+$@}" + ;; + --documentation|--doc|-d) + shift 1 + import::list_modules "import::show_documentation 1" "${@:+$@}" | less + ;; + --man) + shift 1 + import::show_documentation 1 "$IMPORT__BIN_FILE" | less + ;; + --help|-h|*) + shift 1 + import::show_documentation 2 "$IMPORT__BIN_FILE" + ;; + esac + fi + +else + # Import given modules when parameters are supplied. + [ "$#" -eq 0 ] || import::import_modules "$@" +fi diff --git a/helpers/helpers.v2.d/bash-modules/log.sh b/helpers/helpers.v2.d/bash-modules/log.sh new file mode 100644 index 000000000..e5afd7375 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/log.sh @@ -0,0 +1,253 @@ +#!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> `log` - various functions related to logging. + +#> +#> ## VARIABLES + +#export PS4='+${BASH_SOURCE}:${LINENO}:${FUNCNAME[0]:+${FUNCNAME[0]}}: '. + +#> * `__log__APP` - name of main file without path. +__log__APP="${IMPORT__BIN_FILE##*/}" # Strip everything before last "/" + +#> * `__log__DEBUG` - set to yes to enable printing of debug messages and stacktraces. +#> * `__log__STACKTRACE` - set to yes to enable printing of stacktraces. + +#> * `__log__TIMESTAMPED` - set to yes to enable timestamped logs +#> * `__log_timestamp_format` - format of timestamp. Default value: "%F %T" (full date and time). +__log_timestamp_format="%F %T" + +#>> +#>> ## FUNCTIONS + +#>> +#>> * `MESSAGE | log::prefix PREFIX` - display string with PREFIX prefixed to every line +#>> +log::prefix() { + __log__PREFIX="$1" + while read -r line; do + # shellcheck disable=SC2001 + echo "$line" | sed 's|^|'"$__log__PREFIX"'|' + done +} + +#>> +#>> * `MESSAGE | log::_generic_log COLOR LEVEL -` - display message with color, date, app name +#>> * or log::_generic_log COLOR LEVEL MESSAGE` +#>> +log::_generic_log() { + local color="$1" ; shift + local level="$1" ; shift + local log_prefix="" log_date="" color_stop="" + if [[ -n "$color" ]]; then + color_stop=$'\033[39m' + fi + if [[ "$#" == 1 ]] && [[ "$1" == "-" ]]; then + while read -r line ; do + if [ "${__log__TIMESTAMPED:-}" == "yes" ]; then + # space at the end + log_date="$(date::print_current_datetime "$__log_timestamp_format") " + fi + log_prefix="${log_date}[$__log__APP] ${color}$level${color_stop}: " + echo "$line" | log::prefix "$log_prefix" + done + else + if [ "${__log__TIMESTAMPED:-}" == "yes" ]; then + # space at the end + log_date="$(date::print_current_datetime "$__log_timestamp_format") " + fi + log_prefix="${log_date}[$__log__APP] ${color}$level${color_stop}: " + echo "$@" | log::prefix "$log_prefix" + fi +} + +#>> +#>> * `stacktrace [INDEX]` - display functions and source line numbers starting +#>> from given index in stack trace, when debugging or back tracking is enabled. +log::stacktrace() { + if [ "${__log__DEBUG:-}" != "yes" ] && [ "${__log__STACKTRACE:-}" != "yes" ]; then + local BEGIN="${1:-1}" # Display line numbers starting from given index, e.g. to skip "log::stacktrace" and "error" functions. + local I + for(( I=BEGIN; I<${#FUNCNAME[@]}; I++ )) + do + echo $'\t\t'"at ${FUNCNAME[$I]}(${BASH_SOURCE[$I]}:${BASH_LINENO[$I-1]})" >&2 + done + echo + fi +} + +#>> +#>> * `log::debug LEVEL MESSAGE...` - print debug-like LEVEL: MESSAGE to STDOUT. +log::debug::custom() { + local LEVEL="${1:-DEBUG}" ; shift + if [ -t 1 ]; then + # STDOUT is tty + local __log_DEBUG_BEGIN=$'\033[96m' + fi + log::_generic_log "${__log_DEBUG_BEGIN:-}" "$LEVEL" "$@" +} + +#>> +#>> * `debug MESAGE...` - print debug message. +log::debug() { + if [ "${__log__DEBUG:-}" == "yes" ]; then + log::debug::custom DEBUG "$@" + fi +} + +#>> +#>> * `log::info LEVEL MESSAGE...` - print info-like LEVEL: MESSAGE to STDOUT. +log::info::custom() { + local LEVEL="${1:-INFO}" ; shift + if [ -t 1 ]; then + # STDOUT is tty + local __log_INFO_BEGIN=$'\033[92m' + fi + log::_generic_log "${__log_INFO_BEGIN:-}" "$LEVEL" "$@" +} + +#>> +#>> * `info MESAGE...` - print info message. +log::info() { + log::info::custom INFO "$@" +} + +#>> +#>> * `log::warn LEVEL MESSAGE...` - print warning-like LEVEL: MESSAGE to STDERR. +log::warn::custom() { + local LEVEL="${1:-WARN}" ; shift + if [ -t 2 ]; then + # STDERR is tty + local __log_WARN_BEGIN=$'\033[93m' + fi + log::_generic_log "${__log_WARN_BEGIN:-}" "$LEVEL" >&2 "$@" +} + +#>> +#>> * `warn MESAGE...` - print warning message and stacktrace (if enabled). +log::warn() { + log::warn::custom WARN "$@" + log::stacktrace 2 +} + +#>> +#>> * `log::error LEVEL MESSAGE...` - print error-like LEVEL: MESSAGE to STDERR. +log::error::custom() { + local LEVEL="$1" ; shift + if [ -t 2 ]; then + # STDERR is tty + local __log_ERROR_BEGIN=$'\033[91m' + fi + log::_generic_log "${__log_ERROR_BEGIN:-}" "$LEVEL" >&2 "$@" +} + +#>> +#>> * `error MESAGE...` - print error message and stacktrace (if enabled). +log::error() { + log::error::custom ERROR "$@" + log::stacktrace 2 +} + +#>> +#>> * `log::fatal LEVEL MESSAGE...` - print a fatal-like LEVEL: MESSAGE to STDERR. +log::fatal::custom() { + local LEVEL="$1" ; shift + if [ -t 2 ]; then + # STDERR is tty + local __log_FATAL_BEGIN=$'\033[95m' + fi + log::_generic_log "${__log_FATAL_BEGIN:-}" "$LEVEL" >&2 "$@" +} + +#>> +#>> * `log::fatal LEVEL MESSAGE...` - print a fatal-like LEVEL: MESSAGE to STDERR. +log::fatal() { + log::fatal::custom FATAL "$@" + log::stacktrace 2 +} + +#>> +#>> * `panic MESAGE...` - print error message and stacktrace, then exit with error code 1. +log::panic() { + log::fatal::custom "PANIC" "$@" + log::enable_stacktrace + log::stacktrace 2 + exit 1 +} + +#>> +#>> * `unimplemented MESSAGE...` - print error message and stacktrace, then exit with error code 42. +log::unimplemented() { + log::fatal::custom "UNIMPLEMENTED" "$@" + log::enable_stacktrace + log::stacktrace 2 + exit 42 +} + +#>> +#>> * `todo MESAGE...` - print todo message and stacktrace. +log::todo() { + log::warn::custom "TODO" "$@" + local __log__STACKTRACE="yes" + log::stacktrace 2 +} + +#>> +#>> * `dbg VARIABLE...` - print name of variable and it content to stderr +log::dbg() { + log::debug "$(declare -p "$@" | sed 's|declare -. ||')" +} + +#>> +#>> * `log::enable_debug_mode` - enable debug messages and stack traces. +log::enable_debug_mode() { + __log__DEBUG="yes" +} + +#>> +#>> * `log::disable_debug_mode` - disable debug messages and stack traces. +log::disable_debug_mode() { + __log__DEBUG="no" +} + +#>> +#>> * `log::enable_stacktrace` - enable stack traces. +log::enable_stacktrace() { + __log__STACKTRACE="yes" +} + +#>> +#>> * `log::disable_stacktrace` - disable stack traces. +log::disable_stacktrace() { + __log__STACKTRACE="no" +} + +#>> +#>> * `log::enable_timestamps` - enable timestamps. +log::enable_timestamps() { + __log__TIMESTAMPED="yes" +} + +#>> +#>> * `log::disable_timestamps` - disable timestamps. +log::disable_timestamps() { + __log__TIMESTAMPED="no" +} + +#>> +#>> * `log::set_timestamp_format FORMAT` - Set format for date. Default value is "%F %T". +log::set_timestamp_format() { + __log_timestamp_format="$1" +} + +#>> +#>> ## NOTES +#>> +#>> - If STDOUT is connected to tty, then +#>> * info and info-like messages will be printed with message level higlighted in green, +#>> * warn and warn-like messages will be printed with message level higlighted in yellow, +#>> * error and error-like messages will be printed with message level higlighted in red. diff --git a/helpers/helpers.v2.d/bash-modules/log_run.sh b/helpers/helpers.v2.d/bash-modules/log_run.sh new file mode 100644 index 000000000..249a794b7 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/log_run.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +# Import log module +import::import_module log + +#>> ## NAME +#>> +#>>> `log_run` - various functions related to logging commands. + +run::stderr_to_stdout() { + "$@" 2>&1 +} + +run::debug() { + "$@" | log::debug - +} + +run::info() { + "$@" | log::info - +} + +run::warn() { + "$@" | log::warn - +} + +run::error() { + "$@" | log::error - +} + +run::fatal() { + "$@" | log::fatal - +} + +run::quiet() { + local result returncode + result=$(run::stderr_to_stdout "$@" || echo "__log_run__returncode=$?") + returncode=$(echo "$result" | sed -n 's|^__log_run__returncode=\(.*\)$|\1|p') + if [[ -n "$returncode" ]]; then + log::error "$@" + return "$returncode" + fi +} diff --git a/helpers/helpers.v2.d/bash-modules/meta.sh b/helpers/helpers.v2.d/bash-modules/meta.sh new file mode 100644 index 000000000..dc9f318c5 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/meta.sh @@ -0,0 +1,89 @@ +##!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> `meta` - functions for working with bash functions. + +#>> +#>> ## FUNCTIONS + +#>> +#>> * `meta::copy_function FUNCTION_NAME NEW_FUNCTION_PREFIX` - copy function to new function with prefix in name. +#> Create copy of function with new prefix. +#> Old function can be redefined or `unset -f`. +meta::copy_function() { + local FUNCTION_NAME="$1" + local PREFIX="$2" + + eval "$PREFIX$(declare -fp $FUNCTION_NAME)" +} + +#>> +#>> * `meta::wrap BEFORE AFTER FUNCTION_NAME[...]` - wrap function. +#> Create wrapper for a function(s). Execute given commands before and after +#> each function. Original function is available as meta::orig_FUNCTION_NAME. +meta::wrap() { + local BEFORE="$1" + local AFTER="$2" + shift 2 + + local FUNCTION_NAME + for FUNCTION_NAME in "$@" + do + # Rename original function + meta::copy_function "$FUNCTION_NAME" "meta::orig_" || return 1 + + # Redefine function + eval " +function $FUNCTION_NAME() { + $BEFORE + + local __meta__EXIT_CODE=0 + meta::orig_$FUNCTION_NAME \"\$@\" || __meta__EXIT_CODE=\$? + + $AFTER + + return \$__meta__EXIT_CODE +} +" + done +} + + +#>> +#>> * `meta::functions_with_prefix PREFIX` - print list of functions with given prefix. +meta::functions_with_prefix() { + compgen -A function "$1" +} + +#>> +#>> * `meta::is_function FUNC_NAME` Checks is given name corresponds to a function. +meta::is_function() { + declare -F "$1" >/dev/null +} + +#>> +#>> * `meta::dispatch PREFIX COMMAND [ARGUMENTS...]` - execute function `PREFIX__COMMAND [ARGUMENTS]` +#> +#> For example, it can be used to execute functions (commands) by name, e.g. +#> `main() { meta::dispatch command__ "$@" ; }`, when called as `man hw world` will execute +#> `command_hw "$world"`. When command handler is not found, dispatcher will try +#> to call `PREFIX__DEFAULT` function instead, or return error code when defaulf handler is not found. +meta::dispatch() { + local prefix="${1:?Prefix is required.}" + local command="${2:?Command is required.}" + shift 2 + + local fn="${prefix}${command}" + + # Is handler function exists? + meta::is_function "$fn" || { + # Is default handler function exists? + meta::is_function "${prefix}__DEFAULT" || { echo "ERROR: Function \"$fn\" is not found." >&2; return 1; } + fn="${prefix}__DEFAULT" + } + + "$fn" "${@:+$@}" || return $? +} diff --git a/helpers/helpers.v2.d/bash-modules/renice.sh b/helpers/helpers.v2.d/bash-modules/renice.sh new file mode 100644 index 000000000..a36b188f0 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/renice.sh @@ -0,0 +1,14 @@ +##!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> `renice` - reduce priority of current shell to make it low priority task (`renice 19` to self). +#>> +#>> ## USAGE +#>> +#>> `. import.sh renice` + +# Run this script as low priority task +renice 19 -p $$ >/dev/null diff --git a/helpers/helpers.v2.d/bash-modules/settings.sh b/helpers/helpers.v2.d/bash-modules/settings.sh new file mode 100644 index 000000000..482ccda8d --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/settings.sh @@ -0,0 +1,54 @@ +##!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ +import::import_modules log arguments + +#>> ## NAME +#>> +#>>> `settings` - import settings from configuration files and configuration directories. +#>> Also known as "configuration directory" pattern. + +#>> +#>> ## FUNCTIONS + +#>> * `settings::import [-e|--ext EXTENSION] FILE|DIRECTORY...` - Import settings +#> (source them into current program as shell script) when +#> file or directory exists. For directories, all files with given extension +#> (`".sh"` by default) are imported, without recursion. +#> +#> **WARNING:** this method is powerful, but unsafe, because user can put any shell +#> command into the configuration file, which will be executed by script. +#> +#> **TODO:** implement file parsing instead of sourcing. +settings::import() { + local __settings_EXTENSION="sh" + arguments::parse '-e|--ext)__settings_EXTENSION;String,Required' -- "$@" || panic "Cannot parse arguments." + + local __settings_ENTRY + for __settings_ENTRY in "${@:+$@}" + do + if [ -f "$__settings_ENTRY" -a -r "$__settings_ENTRY" -a -s "$__settings_ENTRY" ] + then + # Just source configuration file into this script. + source "$__settings_ENTRY" || { + error "Cannot import settings from \"$__settings_ENTRY\" file: non-zero exit code returned: $?." >&2 + return 1 + } + elif [ -d "$__settings_ENTRY" -a -x "$__settings_ENTRY" ] + then + # Just source each configuration file in the directory into this script. + local __settings_FILE + for __settings_FILE in "$__settings_ENTRY"/*."$__settings_EXTENSION" + do + if [ -f "$__settings_FILE" -a -r "$__settings_FILE" -a -s "$__settings_FILE" ] + then + source "$__settings_FILE" || { + error "Cannot import settings from \"$__settings_FILE\" file: non-zero exit code returned: $?." >&2 + return 1 + } + fi + done + fi + done + return 0 +} diff --git a/helpers/helpers.v2.d/bash-modules/strict.sh b/helpers/helpers.v2.d/bash-modules/strict.sh new file mode 100644 index 000000000..a39f97107 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/strict.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +import::import_module array + +#>> ## NAME +#>> +#>>> `strict` - unofficial strict mode for bash +#>> +#>> Just import this module, to enabe strict mode: `set -euEo pipefail`. +#> +#> ## NOTE +#> +#> * Option `-e` is not working when command is part of a compound command, +#> or in subshell. See bash manual for details. For example, `-e` may not working +#> in a `for` cycle. + +set -euEo pipefail + +declare -Ag __cleanup_CODES + +cleanup::run() { + for key in "${!__cleanup_CODES[@]}"; do + echo "Cleaning up $key..." + cleanup::pop "$key" + done +} + +cleanup::add() { + key="$1" ; shift + value="${1:-}" + __cleanup_CODES+=([$key]="$value") +} + +cleanup::remove() { + local key="$1" + unset "__cleanup_CODES[$key]" +} + +cleanup::pop() { + local key="$1" + if array::contains "$key" __cleanup_CODES; then + code="${__cleanup_CODES[$key]}" + cleanup::remove "$key" + eval "$code" + fi +} + +trap 'log::panic "Uncaught error."' ERR +trap 'cleanup::run' EXIT diff --git a/helpers/helpers.v2.d/bash-modules/string.sh b/helpers/helpers.v2.d/bash-modules/string.sh new file mode 100644 index 000000000..6f4834877 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/string.sh @@ -0,0 +1,274 @@ +##!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> string - various functions to manipulate strings. + +#>> +#>> ## FUNCTIONS + +#>> +#>> * `string::trim_spaces VARIABLE VALUE` +#> Trim white space characters around value and assign result to variable. +string::trim() { + local -n __string__VAR="$1" + local __string__VALUE="${2:-}" + + # remove leading whitespace characters + __string__VALUE="${__string__VALUE#"${__string__VALUE%%[![:space:]]*}"}" + # remove trailing whitespace characters + __string__VALUE="${__string__VALUE%"${__string__VALUE##*[![:space:]]}"}" + + __string__VAR="$__string__VALUE" +} + +#>> +#>> * `string::trim_start VARIABLE VALUE` +#> Trim white space characters at begining of the value and assign result to the variable. +string::trim_start() { + local -n __string__VAR="$1" + local __string__VALUE="${2:-}" + + # remove leading whitespace characters + __string__VALUE="${__string__VALUE#"${__string__VALUE%%[![:space:]]*}"}" #" + + __string__VAR="$__string__VALUE" +} + +#>> +#>> * `string::trim_end VARIABLE VALUE` +#> Trim white space characters at the end of the value and assign result to the variable. +string::trim_end() { + local -n __string__VAR="$1" + local __string__VALUE="${2:-}" + + # remove trailing whitespace characters + __string__VALUE="${__string__VALUE%"${__string__VALUE##*[![:space:]]}"}" #" + + __string__VAR="$__string__VALUE" +} + +#>> +#>> * `string::insert VARIABLE POSITION VALUE` +#> Insert `VALUE` into `VARIABLE` at given `POSITION`. +#> Example: +#> +#> ```bash +#> v="abba" +#> string::insert v 2 "cc" +#> # now v=="abccba" +#> ``` +string::insert() { + local -n __string__VAR="$1" + local __string__POSITION="$2" + local __string__VALUE="${3:-}" + + __string__VALUE="${__string__VAR::$__string__POSITION}${__string__VALUE}${__string__VAR:$__string__POSITION}" + + __string__VAR="$__string__VALUE" +} + +#>> +#>> * `string::split ARRAY DELIMITERS VALUE` +#> Split value by delimiter(s) and assign result to array. Use +#> backslash to escape delimiter in string. +string::split_to() { + local __string__VAR="$1" + local IFS="$2" + local __string__VALUE="${3:-}" + + # We can use "for" loop and strip elements item by item, but we are + # unable to assign result to named array, so we must use "read -a" and "<<<" here. + + # TODO: use regexp and loop instead. + read -a "$__string__VAR" <<<"${__string__VALUE:-}" +} + +#>> +#>> * `string::split DELIMITERS VALUE` +#> Split value by delimiter(s) and echo the result. Use +#> backslash to escape delimiter in string. +string::split() { + local -a __string__ECHO + string::split_to "__string__ECHO" "$@" + echo "${__string__ECHO[@]}" +} + +#>> +#>> * `string::basename VARIABLE FILE [EXT]` +#> Strip path and optional extension from full file name and store +#> file name in variable. +string::basename() { + local -n __string__VAR="$1" + local __string__FILE="${2:-}" + local __string__EXT="${3:-}" + + __string__FILE="${__string__FILE##*/}" # Strip everything before last "/" + __string__FILE="${__string__FILE%$__string__EXT}" # Strip .sh extension + + __string__VAR="$__string__FILE" +} + +#>> +#>> * `string::dirname VARIABLE FILE` +#> Strip file name from path and store directory name in variable. +string::dirname() { + local -n __string__VAR="$1" + local __string__FILE="${2:-}" + + local __string__DIR="" + case "$__string__FILE" in + */*) + __string__DIR="${__string__FILE%/*}" # Strip everything after last "/' + ;; + *) + __string__DIR="." + ;; + esac + + __string__VAR="$__string__DIR" +} + +#>> +#>> * `string::random_string VARIABLE LENGTH` +#> Generate random string of given length using [a-zA-Z0-9] +#> characters and store it into variable. +string::random_string() { + local -n __string__VAR="$1" + local __string__LENGTH="${2:-8}" + + local __string__ALPHABET="0123456789qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM" + local __string__ALPHABET_LENGTH=${#__string__ALPHABET} + + local __string__I __string__RESULT="" + for((__string__I=0; __string__I<__string__LENGTH; __string__I++)) + do + __string__RESULT="$__string__RESULT${__string__ALPHABET:RANDOM%__string__ALPHABET_LENGTH:1}" + done + + __string__VAR="$__string__RESULT" +} + +#>> +#>> * `string::chr VARIABLE CHAR_CODE` +#> Convert decimal character code to its ASCII representation. +string::chr() { + local __string__VAR="$1" + local __string__CODE="$2" + + local __string__OCTAL_CODE + printf -v __string__OCTAL_CODE '%03o' "$__string__CODE" + printf -v "$__string__VAR" "\\$__string__OCTAL_CODE" +} + +#>> +#>> * `string::ord VARIABLE CHAR` +#> Converts ASCII character to its decimal value. +string::ord() { + local __string__VAR="$1" + local __string__CHAR="$2" + + printf -v "$__string__VAR" '%d' "'$__string__CHAR" +} + +# Alternative version of function: +# string::quote_to_bash_format() { +# local -n __string__VAR="$1" +# local __string__STRING="$2" +# +# local __string__QUOTE="'\\''" +# local __string__QUOTE="'\"'\"'" +# __string__VAR="'${__string__STRING//\'/$__string__QUOTE}'" +# } + +#>> +#>> * `string::quote_to_bash_format VARIABLE STRING` +#> Quote the argument in a way that can be reused as shell input. +string::quote_to_bash_format() { + local __string__VAR="$1" + local __string__STRING="$2" + + printf -v "$__string__VAR" "%q" "$__string__STRING" +} + +#>> +#>> * `string::unescape_backslash_sequences VARIABLE STRING` +#> Expand backslash escape sequences. +string::unescape_backslash_sequences() { + local __string__VAR="$1" + local __string__STRING="$2" + + printf -v "$__string__VAR" "%b" "$__string__STRING" +} + +#>> +#>> * `string::to_identifier VARIABLE STRING` +#> Replace all non-alphanumeric characters in string by underscore character. +string::to_identifier() { + local -n __string__VAR="$1" + local __string__STRING="$2" + + # We need a-zA-Z letters only. + # 'z' can be in middle of alphabet on some locales. + __string__VAR="${__string__STRING//[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789]/_}" +} + +#>> +#>> * `string::find_string_with_prefix VAR PREFIX [STRINGS...]` +#> Find first string with given prefix and assign it to VAR. +string::find_string_with_prefix() { + local -n __string__VAR="$1" + local __string__PREFIX="$2" + shift 2 + + local __string__I + for __string__I in "$@" + do + [[ "${__string__I}" != "${__string__PREFIX}"* ]] || { + __string__VAR="${__string__I}" + return 0 + } + done + return 1 +} + +#>> +#>> * `string::empty STRING` +#> Returns zero exit code (true), when string is empty +string::empty() { + [[ -z "${1:-}" ]] +} + + + +#>> +#>> * `string::contains STRING SUBSTRING` +#> Returns zero exit code (true), when string contains substring +string::contains() { + case "$1" in + *"$2"*) return 0 ;; + *) return 1 ;; + esac +} + +#>> +#>> * `string::starts_with STRING SUBSTRING` +#> Returns zero exit code (true), when string starts with substring +string::starts_with() { + case "$1" in + "$2"*) return 0 ;; + *) return 1 ;; + esac +} + +#>> +#>> * `string::ends_with STRING SUBSTRING` +#> Returns zero exit code (true), when string ends with substring +string::ends_with() { + case "$1" in + *"$2") return 0 ;; + *) return 1 ;; + esac +} diff --git a/helpers/helpers.v2.d/bash-modules/timestamped_log.sh b/helpers/helpers.v2.d/bash-modules/timestamped_log.sh new file mode 100644 index 000000000..0d85051f9 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/timestamped_log.sh @@ -0,0 +1,52 @@ +##!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +# Import log module and then override some functions +import::import_module log date meta + +#>> ## NAME +#>> +#>>> `timestamped_log` - print timestamped logs. Drop-in replacement for `log` module. + +#> +#> ## VARIABLES + +#> +#> * `__timestamped_log_format` - format of timestamp. Default value: "%F %T" (full date and time). +__timestamped_log_format="%F %T " + + +#>> +#>> ## FUNCTIONS + +#>> +#>> * `timestamped_log::set_format FORMAT` - Set format for date. Default value is "%F %T". +timestamped_log::set_format() { + __timestamped_log_format="$1" +} + +#>> +#>> ## Wrapped functions: +#>> +#>> `log::info`, `info`, `debug` - print timestamp to stdout and then log message. +meta::wrap \ + 'date::print_current_datetime "$__timestamped_log_format"' \ + '' \ + log::info::custom \ + log::debug + +#>> +#>> `log::error`, `log::warn`, `error`, `warn` - print timestamp to stderr and then log message. +meta::wrap \ + 'date::print_current_datetime "$__timestamped_log_format" >&2' \ + '' \ + log::warn::custom \ + log::error::custom \ + log::fatal::custom + +#>> +#>> ## NOTES +#>> +#>> See `log` module usage for details about log functions. Original functions +#>> are available with prefix `"timestamped_log::orig_"`. diff --git a/helpers/helpers.v2.d/bash-modules/unit.sh b/helpers/helpers.v2.d/bash-modules/unit.sh new file mode 100644 index 000000000..17b1d20c4 --- /dev/null +++ b/helpers/helpers.v2.d/bash-modules/unit.sh @@ -0,0 +1,259 @@ +##!/bin/bash +# Copyright (c) 2009-2021 Volodymyr M. Lisivka , All Rights Reserved +# License: LGPL2+ + +#>> ## NAME +#>> +#>>> `unit` - functions for unit testing. + +import::import_module log arguments + +#>> +#>> ## FUNCTIONS + +#>> +#>> * `unit::assert_yes VALUE [MESSAGE]` - Show error message, when `VALUE` is not equal to `"yes"`. +unit::assert_yes() { + local VALUE="${1:-}" + local MESSAGE="${2:-Value is not \"yes\".}" + + [ "${VALUE:-}" == "yes" ] || { + log::error::custom "ASSERT FAILED" "$MESSAGE" + exit 1 + } +} + +#>> +#>> * `unit::assert_no VALUE [MESSAGE]` - Show error message, when `VALUE` is not equal to `"no"`. +unit::assert_no() { + local VALUE="$1" + local MESSAGE="${2:-Value is not \"no\".}" + + [ "$VALUE" == "no" ] || { + log::error::custom "ASSERT FAILED" "$MESSAGE" + exit 1 + } +} + +#>> +#>> * `unit::assert_not_empty VALUE [MESSAGE]` - Show error message, when `VALUE` is empty. +unit::assert_not_empty() { + local VALUE="${1:-}" + local MESSAGE="${2:-Value is empty.}" + + [ -n "${VALUE:-}" ] || { + log::error::custom "ASSERT FAILED" "$MESSAGE" + exit 1 + } +} + +#>> +#>> * `unit::assert_equal ACTUAL EXPECTED [MESSAGE]` - Show error message, when values are not equal. +unit::assert_equal() { + local ACTUAL="${1:-}" + local EXPECTED="${2:-}" + local MESSAGE="${3:-Values are not equal.}" + + [ "${ACTUAL:-}" == "${EXPECTED:-}" ] || { + log::error::custom "ASSERT FAILED" "$MESSAGE Actual value: \"${ACTUAL:-}\", expected value: \"${EXPECTED:-}\"." + exit 1 + } +} + +#>> +#>> * `unit::assert_arrays_are_equal MESSAGE VALUE1... -- VALUE2...` - Show error message when arrays are not equal in size or content. +unit::assert_arrays_are_equal() { + local MESSAGE="${1:-Arrays are not equal.}" ; shift + local ARGS=( $@ ) + + local I LEN1='' + for((I=0;I<${#ARGS[@]};I++)) + do + [ "${ARGS[I]}" != "--" ] || { + LEN1="$I" + break + } + done + + [ -n "${LEN1:-}" ] || { + error "Array separator is not found. Put \"--\" between two arrays." + exit 1 + } + + local LEN2=$(($# - LEN1 - 1)) + local MIN=$(( (LEN1> +#>> * `unit::assert_not_equal ACTUAL_VALUE UNEXPECTED_VALUE [MESSAGE]` - Show error message, when values ARE equal. +unit::assert_not_equal() { + local ACTUAL_VALUE="${1:-}" + local UNEXPECTED_VALUE="${2:-}" + local MESSAGE="${3:-values are equal but must not.}" + + [ "${ACTUAL_VALUE:-}" != "${UNEXPECTED_VALUE:-}" ] || { + log::error::custom "ASSERT FAILED" "$MESSAGE Actual value: \"${ACTUAL_VALUE:-}\", unexpected value: \"$UNEXPECTED_VALUE\"." + exit 1 + } +} + +#>> +#>> * `unit::assert MESSAGE TEST[...]` - Evaluate test and show error message when it returns non-zero exit code. +unit::assert() { + local MESSAGE="${1:-}"; shift + + eval "$@" || { + log::error::custom "ASSERT FAILED" "${MESSAGE:-}: $@" + exit 1 + } +} + +#>> +#>> * `unit::fail [MESSAGE]` - Show error message. +unit::fail() { + local MESSAGE="${1:-This point in test case must not be reached.}"; shift + log::error::custom "FAIL" "$MESSAGE $@" + exit 1 +} + +#>> +#>> * `unit::run_test_cases [OPTIONS] [--] [ARGUMENTS]` - Execute all functions with +#>> test* prefix in name in alphabetic order +#> +#> * OPTIONS: +#> * `-t | --test TEST_CASE` - execute single test case, +#> * `-q | --quiet` - do not print informational messages and dots, +#> * `--debug` - enable stack traces. +#> * ARGUMENTS - All arguments, which are passed to run_test_cases, are passed then +#> to `unit::set_up`, `unit::tear_down` and test cases using `ARGUMENTS` array, so you +#> can parametrize your test cases. You can call `run_test_cases` more than +#> once with different arguments. Use `"--"` to strictly separate arguments +#> from options. +#> +#> After execution of `run_test_cases`, following variables will have value: +#> +#> * `NUMBER_OF_TEST_CASES` - total number of test cases executed, +#> * `NUMBER_OF_FAILED_TEST_CASES` - number of failed test cases, +#> * `FAILED_TEST_CASES` - names of functions of failed tests cases. +#> +#> +#> If you want to ignore some test case, just prefix them with +#> underscore, so `unit::run_test_cases` will not see them. +#> +#> If you want to run few subsets of test cases in one file, define each +#> subset in it own subshell and execute `unit::run_test_cases` in each subshell. +#> +#> Each test case is executed in it own subshell, so you can call `exit` +#> in the test case or assign variables without any effect on subsequent test +#> cases. +unit::run_test_cases() { + + NUMBER_OF_TEST_CASES=0 + NUMBER_OF_FAILED_TEST_CASES=0 + FAILED_TEST_CASES=( ) + + local __QUIET=no __TEST_CASES=( ) + + arguments::parse \ + "-t|test)__TEST_CASES;Array" \ + "-q|--quiet)__QUIET;Yes" \ + -- "$@" || panic "Cannot parse arguments. Arguments: $*" + + # If no test cases are given via options + [ "${#__TEST_CASES[@]}" -gt 0 ] || { + # Then generate list of test cases using compgen + # As alternative, declare -F | cut -d ' ' -f 3 | grep '^test' can be used + __TEST_CASES=( $(compgen -A function test) ) || panic "No test cases are found. Create a function with test_ prefix in the name." + } + + local __TEST __EXIT_CODE=0 + + ( set -ueEo pipefail ; FIRST_TEAR_DOWN=yes ; unit::tear_down "${ARGUMENTS[@]:+${ARGUMENTS[@]}}" ) || { + __EXIT_CODE=$? + log::error::custom "FAIL" "tear_down before first test case is failed." + } + + for __TEST in "${__TEST_CASES[@]:+${__TEST_CASES[@]}}" + do + let NUMBER_OF_TEST_CASES++ || : + [ "$__QUIET" == "yes" ] || echo -n "." + + ( + __EXIT_CODE=0 + + unit::set_up "${ARGUMENTS[@]:+${ARGUMENTS[@]}}" || { + __EXIT_CODE=$? + unit::fail "unit::set_up failed before test case #$NUMBER_OF_TEST_CASES ($__TEST)." + } + + ( "$__TEST" "${ARGUMENTS[@]:+${ARGUMENTS[@]}}" ) || { + __EXIT_CODE=$? + unit::fail "Test case #$NUMBER_OF_TEST_CASES ($__TEST) failed." + } + + unit::tear_down "${ARGUMENTS[@]:+${ARGUMENTS[@]}}" || { + __EXIT_CODE=$? + unit::fail "unit::tear_down failed after test case #$NUMBER_OF_TEST_CASES ($__TEST)." + } + exit $__EXIT_CODE # Exit from subshell + ) || { + __EXIT_CODE=$? + let NUMBER_OF_FAILED_TEST_CASES++ || : + FAILED_TEST_CASES[${#FAILED_TEST_CASES[@]}]="$__TEST" + } + done + + [ "$__QUIET" == "yes" ] || echo + if [ "$__EXIT_CODE" -eq 0 ] + then + [ "$__QUIET" == "yes" ] || log::info "OK" "Test cases total: $NUMBER_OF_TEST_CASES, failed: $NUMBER_OF_FAILED_TEST_CASES${FAILED_TEST_CASES[@]:+, failed methods: ${FAILED_TEST_CASES[@]}}." + else + log::error::custom "FAIL" "Test cases total: $NUMBER_OF_TEST_CASES, failed: $NUMBER_OF_FAILED_TEST_CASES${FAILED_TEST_CASES[@]:+, failed methods: ${FAILED_TEST_CASES[@]}}." + fi + + return $__EXIT_CODE +} + +#> +#> `unit::run_test_cases` will also call `unit::set_up` and `unit::tear_down` +#> functions before and after each test case. By default, they do nothing. +#> Override them to do something useful. + +#>> +#>> * `unit::set_up` - can set variables which are available for following +#>> test case and `tear_down`. It also can alter `ARGUMENTS` array. Test case +#>> and tear_down are executed in their own subshell, so they cannot change +#>> outer variables. +unit::set_up() { + return 0 +} + +#>> +#>> * `unit::tear_down` is called first, before first set_up of first test case, to +#>> cleanup after possible failed run of previous test case. When it +#>> called for first time, `FIRST_TEAR_DOWN` variable with value `"yes"` is +#>> available. +unit::tear_down() { + return 0 +} + + +#> +#> ## NOTES +#> +#> All assert functions are executing `exit` instead of returning error code. diff --git a/helpers/helpers.v2.d/fail2ban b/helpers/helpers.v2.d/fail2ban new file mode 100644 index 000000000..5c8d0d074 --- /dev/null +++ b/helpers/helpers.v2.d/fail2ban @@ -0,0 +1,137 @@ +#!/bin/bash + +# Create a dedicated fail2ban config (jail and filter conf files) +# +# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# +# ----------------------------------------------------------------------------- +# +# usage 2: ynh_add_fail2ban_config --use_template +# | arg: -t, --use_template - Use this helper in template mode +# +# This will use a template in `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf` +# See the documentation of `ynh_add_config` for a description of the template +# format and how placeholders are replaced with actual variables. +# +# Generally your template will look like that by example (for synapse): +# ``` +# f2b_jail.conf: +# [__APP__] +# enabled = true +# port = http,https +# filter = __APP__ +# logpath = /var/log/__APP__/logfile.log +# maxretry = 3 +# ``` +# ``` +# f2b_filter.conf: +# [INCLUDES] +# before = common.conf +# [Definition] +# +# # Part of regex definition (just used to make more easy to make the global regex) +# __synapse_start_line = .? \- synapse\..+ \- +# +# # Regex definition. +# failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ +# +# ignoreregex = +# ``` +# +# ----------------------------------------------------------------------------- +# +# Note about the "failregex" option: +# +# regex to match the password failure messages in the logfile. The host must be +# matched by a group named "`host`". The tag "``" can be used for standard +# IP/hostname matching and is only an alias for `(?:::f{4,6}:)?(?P[\w\-.^_]+)` +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# Note that the logfile need to exist before to call this helper !! +# +# To validate your regex you can test with this command: +# ``` +# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf +# ``` +# +# Requires YunoHost version 4.1.0 or higher. + +__FAIL2BAN_FILTER_TEMPLATE="[INCLUDES] +before = common.conf +[Definition] +failregex = __FAILREGEX__ +ignoreregex = +" + +__FAIL2BAN_JAIL_TEMPLATE="[__APP__] +enabled = true +port = __PORTS__ +filter = __APP__ +logpath = __LOGPATH__ +maxretry = __MAX_RETRY__ +" + + +ynh::fail2ban::add() { + local logpath + local failregex + local max_retry=3 + local ports="http,https" + local jail_template filter_template + arguments::parse \ + "-l|--logpath)logpath;String" \ + "-r|--failregex)failregex;String" \ + "-m|--max_retries)max_retries;String" \ + "-p|--ports)ports;String" \ + "-t|--jail_template)jail_template;String" \ + "-t|--filter_template)filter_template;String" \ + -- "$@" + + if string::empty "$filter_template"; then + filter_template="fail2ban_filter.conf" + # Mandatory arguments + if string::empty "failregex"; then + log::panic "No failregex was passed to ${FUNCNAME[0]}" + fi + echo "$__FAIL2BAN_FILTER_TEMPLATE" > "$YNH_APP_BASEDIR/conf/$filter_template" + fi + + if string::empty "$jail_template"; then + jail_template="fail2ban_jail.conf" + # Mandatory arguments + if string::empty "logpath"; then + log::panic "No logpath was passed to ${FUNCNAME[0]}" + fi + echo "$__FAIL2BAN_JAIL_TEMPLATE" > "$YNH_APP_BASEDIR/conf/$jail_template" + fi + + ynh::config::add --template="$filter_template" --destination="/etc/fail2ban/filter.d/$app.conf" + ynh::config::add --template="$jail_template" --destination="/etc/fail2ban/jail.d/$app.conf" + + ynh::system::action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + + local fail2ban_error + fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" + if ! string::empty "$fail2ban_error"; then + log::err "Fail2ban failed to load the jail for ${app}:" + log::warn "${fail2ban_error#*WARNING}" + fi + +} + +# Remove the dedicated fail2ban config (jail and filter conf files) +# +# usage: ynh_remove_fail2ban_config +# +# Requires YunoHost version 3.5.0 or higher. +ynh::fail2ban::remove() { + ynh::fs::remove --file="/etc/fail2ban/filter.d/$app.conf" + ynh::fs::remove --file="/etc/fail2ban/jail.d/$app.conf" + ynh::systemd::action --service_name=fail2ban --action=reload +} diff --git a/helpers/helpers.v2.d/import_bash-modules b/helpers/helpers.v2.d/import_bash-modules new file mode 100644 index 000000000..91bc06c31 --- /dev/null +++ b/helpers/helpers.v2.d/import_bash-modules @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +_modules=( + arguments + array + # cd_to_bindir + date + log + log_run + meta + # renice + settings + strict + string + # timestamped_log + # unit +) + +BASH_MODULES_PATH="$YNH_APP_HELPERS_DIR/bash-modules" +source "$BASH_MODULES_PATH/import.sh" "${_modules[@]}" diff --git a/helpers/helpers.v2.d/setting b/helpers/helpers.v2.d/setting new file mode 100644 index 000000000..d961c2ce1 --- /dev/null +++ b/helpers/helpers.v2.d/setting @@ -0,0 +1,151 @@ +#!/bin/bash + +# Small "hard-coded" interface to avoid calling "yunohost app" directly each +# time dealing with a setting is needed (which may be so slow on ARM boards) +# +# [internal] +# +ynh::_setting() { + python3 -c "$__YNH_SETTING_PYTHON_ACCESS" "$1" "$2" "$3" "${4:-}" +} +__YNH_SETTING_PYTHON_ACCESS=$(cat << EOF +import os, sys, yaml +_, app, action, key, value = sys.argv +action = action.lower() + +setting_file = "/etc/yunohost/apps/%s/settings.yml" % app +assert os.path.exists(setting_file), "Setting file %s does not exists ?" % setting_file +with open(setting_file) as f: + settings = yaml.safe_load(f) +if action == "get": + if key in settings: + print(settings[key]) +else: + if action == "delete": + if key in settings: + del settings[key] + elif action == "set": + if key in ['redirected_urls', 'redirected_regex']: + value = yaml.safe_load(value) + settings[key] = value + else: + raise ValueError("action should either be get, set or delete") + with open(setting_file, "w") as f: + yaml.safe_dump(settings, f, default_flow_style=False) +EOF +) + +# Get an application setting +# +# usage: ynh::setting::get [--app=app] --key=key +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting to get +# +# Requires YunoHost version 2.2.4 or higher. +ynh::setting::get() { + local app="$YNH_APP_ID" + local key + arguments::parse \ + "-a|--app)app;String" \ + "-k|--key)key;String,R" \ + -- "$@" + + if string::starts_with "$key" unprotected_ \ + || string::starts_with "$key" protected_ \ + || string::starts_with "$key" skipped_; then + yunohost app setting "$app" "$key" + else + ynh::_setting get "$app" "$key" + fi +} + +# Set an application setting +# +# usage: ynh_app_setting_set --app=app --key=key --value=value +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting name to set +# | arg: -v, --value= - the setting value to set +# +# Requires YunoHost version 2.2.4 or higher. +ynh::setting::set() { + local app="$YNH_APP_ID" + local key value + arguments::parse \ + "-a|--app)app;String" \ + "-k|--key)key;String,R" \ + "-v|--value)value;String,R" \ + -- "$@" + + if string::starts_with "$key" unprotected_ \ + || string::starts_with "$key" protected_ \ + || string::starts_with "$key" skipped_; then + yunohost app setting "$app" "$key" -v "$value" + else + ynh::_setting set "$app" "$key" + fi +} + +# Get an application setting +# +# usage: ynh::setting::get [--app=app] --key=key +# | arg: -a, --app= - the application id +# | arg: -k, --key= - the setting to get +# +# Requires YunoHost version 2.2.4 or higher. +ynh::setting::delete() { + local app="$YNH_APP_ID" + local key + arguments::parse \ + "-a|--app)app;String" \ + "-k|--key)key;String,R" \ + -- "$@" + + if string::starts_with "$key" unprotected_ \ + || string::starts_with "$key" protected_ \ + || string::starts_with "$key" skipped_; then + yunohost app setting "$app" "$key" -d + else + ynh::_setting delete "$app" "$key" + fi +} + +# Check availability of a web path +# +# usage: ynh_webpath_available --domain=domain --path_url=path +# | arg: -d, --domain= - the domain/host of the url +# | arg: -p, --path_url= - the web path to check the availability of +# +# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee +# +# Requires YunoHost version 2.6.4 or higher. +ynh::webpath::is_available() { + local domain path + arguments::parse \ + "-d|--domain)domain;String,R" \ + "-p|--path)path;String,R" \ + -- "$@" + + yunohost domain url-available "$domain" "$path" +} + +# Register/book a web path for an app +# +# usage: ynh_webpath_register --app=app --domain=domain --path_url=path +# | arg: -a, --app= - the app for which the domain should be registered +# | arg: -d, --domain= - the domain/host of the web path +# | arg: -p, --path_url= - the web path to be registered +# +# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee +# +# Requires YunoHost version 2.6.4 or higher. +ynh::webpath::register() { + local app="$YNH_APP_ID" + local domain path + arguments::parse \ + "-a|--app)app;String" \ + "-d|--domain)domain;String,R" \ + "-p|--path)path;String,R" \ + -- "$@" + + yunohost domain register-url "$app" "$domain" "$path" +} diff --git a/helpers/helpers.v2.d/systemd b/helpers/helpers.v2.d/systemd new file mode 100644 index 000000000..b7d7c7419 --- /dev/null +++ b/helpers/helpers.v2.d/systemd @@ -0,0 +1,174 @@ +#!/bin/bash + +# Create a dedicated systemd config +# +# usage: ynh_add_systemd_config [--service=service] [--template=template] +# | arg: -s, --service= - Service name (optionnal, `$app` by default) +# | arg: -t, --template= - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template) +# +# This will use the template `../conf/.service`. +# +# See the documentation of `ynh_add_config` for a description of the template +# format and how placeholders are replaced with actual variables. +# +# Requires YunoHost version 4.1.0 or higher. + +ynh::systemd::add_service() { + local service="$app" + local template=systemd.service + arguments::parse \ + "-s|--service)service;String" \ + "-t|--template)template;String" \ + -- "$@" + + ynh::config::add --template="$template" --destination="/etc/systemd/system/$service.service" + systemctl daemon-reload + systemctl enable "$service" --quiet +} + +# Remove the dedicated systemd config +# +# usage: ynh_remove_systemd_config [--service=service] +# | arg: -s, --service= - Service name (optionnal, $app by default) +# +# Requires YunoHost version 2.7.2 or higher. +ynh::systemd::remove_service() { + local service="$app" + arguments::parse \ + "-s|--service)service;String" \ + -- "$@" + + local service_file="/etc/systemd/system/$service.service" + if [ -e "$service_file" ]; then + ynh_systemd_action --service="$service" --action=stop + systemctl disable "$service" --quiet + ynh::fs::remove --file="$service_file" + systemctl daemon-reload + fi +} + +# Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started +# +# usage: ynh_systemd_action [--service_name=service_name] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] +# | arg: -n, --service_name= - Name of the service to start. Default : `$app` +# | arg: -a, --action= - Action to perform with systemctl. Default: start +# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. +# | arg: -p, --log_path= - Log file - Path to the log file. Default : `/var/log/$app/$app.log` +# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. +# | arg: -e, --length= - Length of the error log displayed for debugging : Default : 20 +# +# Requires YunoHost version 3.5.0 or higher. +ynh::systemd::action() { + local service="$app" + local action + local line_match + local log_path="/var/log/$app/$app.log" + local timeout=300 + local length=20 + arguments::parse \ + "-s|--service)service;String" \ + "-a|--action)action;String,R" \ + "-m|--line_match)line_match;String" \ + "-p|--log_path)log_path;String" \ + "-t|--timeout)timeout;String" \ + "-l|--length)length;String" \ + + + # Manage case of service already stopped + if [ "$action" == "stop" ] && ! systemctl is-active --quiet "$service"; then + return 0 + fi + + # Start to read the log + if ! string::empty "$line_match"; then + local templog + templog="$(mktemp)" + cleanup::add "Systemd action log file" "rm -f \"$templog\"" + # Following the starting of the app in its log + if [ "$log_path" == "systemd" ]; then + # Read the systemd journal + journalctl --unit="$service" --follow --since=-0 --quiet >"$templog" & + local pid_tail=$! + else + # Read the specified log file + tail --follow=name --retry --lines=0 "$log_path" >"$templog" 2>&1 & + local pid_tail=$! + fi + cleanup::add "Systemd action log process" "kill -SIGTERM $pid_tail" + fi + + # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. + if [ "$action" == "reload" ]; then + action="reload-or-restart" + fi + + local time_start + time_start="$(date --utc --rfc-3339=seconds | cut -d+ -f1) UTC" + + # If the service fails to perform the action + if ! systemctl "$action" "$service"; then + # Show syslog for this service + run::error journalctl --quiet --no-hostname --no-pager --lines="$length" --unit="$service" + # If a log is specified for this service, show also the content of this log + if ! string::empty "$log_path"; then + run::error tail --lines=$length "$log_path" + fi + ynh::systemd::_action_cleanup + return 1 + fi + + if ! string::empty "$line_match"; then + local start_time max_time long_time is_long timed_out + start_time=$(date +%s) + long_time=$(( start_time + 30 )) + max_time=$(( start_time + timeout )) + set +x + while true; do + # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout + if [ "$log_path" == "systemd" ]; then + # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action + if journalctl --unit="$service" --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then + log::info "The service $service has correctly executed the action $action." + break + fi + else + if grep --extended-regexp --quiet "$line_match" "$templog"; then + log::info "The service $service has correctly executed the action $action." + break + fi + fi + + if string::empty "$is_long" && (( $(date +%s) >= long_time )); then + is_long=true + log::warn "(this may take some time)" + fi + if (( $(date +%s) < max_time )); then + timed_out=true + fi + sleep 1 + done + set -x + + if [[ "$timed_out" == "true" ]]; then + log::warn "The service $service didn't fully executed the action ${action} before the timeout.\n" + log::warn "Please find here an extract of the end of the log of the service $service:" + run::warn journalctl --quiet --no-hostname --no-pager --lines="$length" --unit="$service" + if [ -e "$log_path" ]; then + log::warn --message="\-\-\-" + run::warn tail --lines=$length "$log_path" + fi + fi + ynh::systemd::_action_cleanup + fi +} + +# Clean temporary process and file used by ynh_check_starting +# +# [internal] +# +# Requires YunoHost version 3.5.0 or higher. + +ynh::systemd::_action_cleanup() { + cleanup::pop "Systemd action log process" + cleanup::pop "Systemd action log file" +} diff --git a/helpers/helpers.v2.d/user b/helpers/helpers.v2.d/user new file mode 100644 index 000000000..f5f3ec7bd --- /dev/null +++ b/helpers/helpers.v2.d/user @@ -0,0 +1,188 @@ +#!/bin/bash + +# Check if a YunoHost user exists +# +# usage: ynh_user_exists --username=username +# | arg: -u, --username= - the username to check +# | ret: 0 if the user exists, 1 otherwise. +# +# example: ynh_user_exists 'toto' || echo "User does not exist" +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null +} + +# Retrieve a YunoHost user information +# +# usage: ynh_user_get_info --username=username --key=key +# | arg: -u, --username= - the username to retrieve info from +# | arg: -k, --key= - the key to retrieve +# | ret: the value associate to that key +# +# example: mail=$(ynh_user_get_info --username="toto" --key=mail) +# +# Requires YunoHost version 2.2.4 or higher. +ynh_user_get_info() { + # Declare an array to define the options of this helper. + local legacy_args=uk + local -A args_array=([u]=username= [k]=key=) + local username + local key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + yunohost user info "$username" --output-as json --quiet | jq -r ".$key" +} + +# Get the list of YunoHost users +# +# usage: ynh_user_list +# | ret: one username per line as strings +# +# example: for u in $(ynh_user_list); do ... ; done +# +# Requires YunoHost version 2.4.0 or higher. +ynh_user_list() { + yunohost user list --output-as json --quiet | jq -r ".users | keys[]" +} + +# Check if a user exists on the system +# +# usage: ynh_system_user_exists --username=username +# | arg: -u, --username= - the username to check +# | ret: 0 if the user exists, 1 otherwise. +# +# Requires YunoHost version 2.2.4 or higher. +ynh_system_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent passwd "$username" &>/dev/null +} + +# Check if a group exists on the system +# +# usage: ynh_system_group_exists --group=group +# | arg: -g, --group= - the group to check +# | ret: 0 if the group exists, 1 otherwise. +# +# Requires YunoHost version 3.5.0.2 or higher. +ynh_system_group_exists() { + # Declare an array to define the options of this helper. + local legacy_args=g + local -A args_array=([g]=group=) + local group + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent group "$group" &>/dev/null +} + +# Create a system user +# +# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell] [--groups="group1 group2"] +# | arg: -u, --username= - Name of the system user that will be create +# | arg: -h, --home_dir= - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home +# | arg: -s, --use_shell - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell +# | arg: -g, --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp) +# +# Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability) : +# ``` +# ynh_system_user_create --username=nextcloud +# ``` +# Create a discourse user using /var/www/discourse as home directory and the default login shell : +# ``` +# ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell +# ``` +# +# Requires YunoHost version 2.6.4 or higher. +ynh_system_user_create() { + # Declare an array to define the options of this helper. + local legacy_args=uhs + local -A args_array=([u]=username= [h]=home_dir= [s]=use_shell [g]=groups=) + local username + local home_dir + local use_shell + local groups + + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + use_shell="${use_shell:-0}" + home_dir="${home_dir:-}" + groups="${groups:-}" + + if ! ynh_system_user_exists "$username"; then # Check if the user exists on the system + # If the user doesn't exist + if [ -n "$home_dir" ]; then # If a home dir is mentioned + local user_home_dir="--home-dir $home_dir" + else + local user_home_dir="--no-create-home" + fi + if [ $use_shell -eq 1 ]; then # If we want a shell for the user + local shell="" # Use default shell + else + local shell="--shell /usr/sbin/nologin" + fi + useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account" + fi + + local group + for group in $groups; do + usermod -a -G "$group" "$username" + done +} + +# Delete a system user +# +# usage: ynh_system_user_delete --username=user_name +# | arg: -u, --username= - Name of the system user that will be create +# +# Requires YunoHost version 2.6.4 or higher. +ynh_system_user_delete() { + # Declare an array to define the options of this helper. + local legacy_args=u + local -A args_array=([u]=username=) + local username + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Check if the user exists on the system + if ynh_system_user_exists "$username"; then + deluser $username + else + ynh_print_warn --message="The user $username was not found" + fi + + # Check if the group exists on the system + if ynh_system_group_exists "$username"; then + delgroup $username + fi +} + +# Execute a command as another user +# +# usage: ynh_exec_as $USER COMMAND [ARG ...] +# +# Requires YunoHost version 4.1.7 or higher. +ynh_exec_as() { + local user=$1 + shift 1 + + if [[ $user = $(whoami) ]]; then + eval "$@" + else + sudo -u "$user" "$@" + fi +}