From c21a45590946feb69f8b699bedb62d25ae2d4a0f Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 17 Mar 2019 22:23:16 +0100 Subject: [PATCH 1/7] Dump ynh log if an app script fails --- data/helpers.d/system | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/data/helpers.d/system b/data/helpers.d/system index fd552db7f..87529c733 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -27,6 +27,32 @@ ynh_exit_properly () { echo -e "!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" >&2 + # Unset xtrace to not spoil the log + set +x + local ynh_log="/var/log/yunohost/yunohost-cli.log" + # Wait for the log to be fill with the data until the crash. + local timeout=0 + while ! tail --lines=20 "$ynh_log" | grep --quiet "ynh_exit_properly" + do + ((timeout++)) + if [ $timeout -eq 500 ]; then + break + fi + done + set -x + + echo -e "\e[34m\e[1mLog extract:\e[0m" >&2 + # Tail the last 30 lines of log of YunoHost + # But remove all lines after "ynh_exit_properly" + # Remove the timestamp at the beginning of the line + # Remove "yunohost.hook..." + # Add DEBUG and color it at the beginning of each log line. + echo -e "$(tail --lines=30 "$ynh_log" \ + | sed '1,/ynh_exit_properly/!d' \ + | sed 's/^[[:digit:]: ,-]*//g' \ + | sed 's/ *yunohost.hook.*\]//g' \ + | sed 's/^/\\e[34m\\e[1m[DEBUG]\\e[0m: /g')" >&2 + if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script. ynh_clean_setup # Call the function to do specific cleaning for the app. fi From f71117b401b5d33f703e31287913c6f7fff94076 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 17 Mar 2019 23:30:57 +0100 Subject: [PATCH 2/7] Support yunohost-api.log --- data/helpers.d/system | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 87529c733..3c065e592 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -1,5 +1,14 @@ #!/bin/bash +# Determine whether the scripts is executed from a terminal or the admin JS. +# +# [internal] +ynh_is_term () { + # Return true if $TERM return xterm-XXXcolor + # Which means the script is executed from a terminal, not from the admin JS + [[ "$TERM" =~ "xterm" ]] +} + # Manage a fail of the script # # [internal] @@ -29,7 +38,13 @@ ynh_exit_properly () { # Unset xtrace to not spoil the log set +x - local ynh_log="/var/log/yunohost/yunohost-cli.log" + + if ynh_is_term + then + local ynh_log="/var/log/yunohost/yunohost-cli.log" + else + local ynh_log="/var/log/yunohost/yunohost-api.log" + fi # Wait for the log to be fill with the data until the crash. local timeout=0 while ! tail --lines=20 "$ynh_log" | grep --quiet "ynh_exit_properly" From fa0e37f8daeb2b13487b1b45173edba24ecbf31c Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 8 May 2019 00:56:18 +0200 Subject: [PATCH 3/7] Print the log only in CLI --- data/helpers.d/system | 308 --------------------------- data/helpers.d/utils | 469 ++++++++++++++++++++++++++++++------------ 2 files changed, 342 insertions(+), 435 deletions(-) delete mode 100644 data/helpers.d/system diff --git a/data/helpers.d/system b/data/helpers.d/system deleted file mode 100644 index 3c065e592..000000000 --- a/data/helpers.d/system +++ /dev/null @@ -1,308 +0,0 @@ -#!/bin/bash - -# Determine whether the scripts is executed from a terminal or the admin JS. -# -# [internal] -ynh_is_term () { - # Return true if $TERM return xterm-XXXcolor - # Which means the script is executed from a terminal, not from the admin JS - [[ "$TERM" =~ "xterm" ]] -} - -# Manage a fail of the script -# -# [internal] -# -# usage: -# ynh_exit_properly is used only by the helper ynh_abort_if_errors. -# You should not use it directly. -# Instead, add to your script: -# ynh_clean_setup () { -# instructions... -# } -# -# This function provide a way to clean some residual of installation that not managed by remove script. -# -# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script -# -ynh_exit_properly () { - local exit_code=$? - if [ "$exit_code" -eq 0 ]; then - exit 0 # Exit without error if the script ended correctly - fi - - trap '' EXIT # Ignore new exit signals - set +eu # Do not exit anymore if a command fail or if a variable is empty - - echo -e "!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" >&2 - - # Unset xtrace to not spoil the log - set +x - - if ynh_is_term - then - local ynh_log="/var/log/yunohost/yunohost-cli.log" - else - local ynh_log="/var/log/yunohost/yunohost-api.log" - fi - # Wait for the log to be fill with the data until the crash. - local timeout=0 - while ! tail --lines=20 "$ynh_log" | grep --quiet "ynh_exit_properly" - do - ((timeout++)) - if [ $timeout -eq 500 ]; then - break - fi - done - set -x - - echo -e "\e[34m\e[1mLog extract:\e[0m" >&2 - # Tail the last 30 lines of log of YunoHost - # But remove all lines after "ynh_exit_properly" - # Remove the timestamp at the beginning of the line - # Remove "yunohost.hook..." - # Add DEBUG and color it at the beginning of each log line. - echo -e "$(tail --lines=30 "$ynh_log" \ - | sed '1,/ynh_exit_properly/!d' \ - | sed 's/^[[:digit:]: ,-]*//g' \ - | sed 's/ *yunohost.hook.*\]//g' \ - | sed 's/^/\\e[34m\\e[1m[DEBUG]\\e[0m: /g')" >&2 - - if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script. - ynh_clean_setup # Call the function to do specific cleaning for the app. - fi - - ynh_die # Exit with error status -} - -# Exits if an error occurs during the execution of the script. -# -# usage: ynh_abort_if_errors -# -# This configure the rest of the script execution such that, if an error occurs -# or if an empty variable is used, the execution of the script stops -# immediately and a call to `ynh_clean_setup` is triggered if it has been -# defined by your script. -# -ynh_abort_if_errors () { - set -eu # Exit if a command fail, and if a variable is used unset. - trap ynh_exit_properly EXIT # Capturing exit signals on shell script -} - -# Fetch the Debian release codename -# -# usage: ynh_get_debian_release -# | ret: The Debian release codename (i.e. jessie, stretch, ...) -ynh_get_debian_release () { - echo $(lsb_release --codename --short) -} - -# Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started -# -# usage: ynh_systemd_action [-n service_name] [-a action] [ [-l "line to match"] [-p log_path] [-t timeout] [-e length] ] -# | arg: -n, --service_name= - Name of the service to start. Default : $app -# | arg: -a, --action= - Action to perform with systemctl. Default: start -# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. -# If not defined it don't wait until the service is completely started. -# WARNING: When using --line_match, you should always add `ynh_clean_check_starting` into your -# `ynh_clean_setup` at the beginning of the script. Otherwise, tail will not stop in case of failure -# of the script. The script will then hang forever. -# | arg: -p, --log_path= - Log file - Path to the log file. Default : /var/log/$app/$app.log -# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. -# | arg: -e, --length= - Length of the error log : Default : 20 -ynh_systemd_action() { - # Declare an array to define the options of this helper. - declare -Ar args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= ) - local service_name - local action - local line_match - local length - local log_path - local timeout - - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - local service_name="${service_name:-$app}" - local action=${action:-start} - local log_path="${log_path:-/var/log/$service_name/$service_name.log}" - local length=${length:-20} - local timeout=${timeout:-300} - - # Start to read the log - if [[ -n "${line_match:-}" ]] - then - local templog="$(mktemp)" - # Following the starting of the app in its log - if [ "$log_path" == "systemd" ] ; then - # Read the systemd journal - journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" & - # Get the PID of the journalctl command - local pid_tail=$! - else - # Read the specified log file - tail -F -n0 "$log_path" > "$templog" 2>&1 & - # Get the PID of the tail command - local pid_tail=$! - fi - fi - - ynh_print_info --message="${action^} the service $service_name" - - # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. - if [ "$action" == "reload" ]; then - action="reload-or-restart" - fi - - systemctl $action $service_name \ - || ( journalctl --no-pager --lines=$length -u $service_name >&2 \ - ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \ - ; false ) - - # Start the timeout and try to find line_match - if [[ -n "${line_match:-}" ]] - then - local i=0 - for i in $(seq 1 $timeout) - do - # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout - if grep --quiet "$line_match" "$templog" - then - ynh_print_info --message="The service $service_name has correctly started." - break - fi - if [ $i -eq 3 ]; then - echo -n "Please wait, the service $service_name is ${action}ing" >&2 - fi - if [ $i -ge 3 ]; then - echo -n "." >&2 - fi - sleep 1 - done - if [ $i -ge 3 ]; then - echo "" >&2 - fi - if [ $i -eq $timeout ] - then - ynh_print_warn --message="The service $service_name didn't fully started before the timeout." - ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" - journalctl --no-pager --lines=$length -u $service_name >&2 - test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 - fi - ynh_clean_check_starting - fi -} - -# Clean temporary process and file used by ynh_check_starting -# (usually used in ynh_clean_setup scripts) -# -# usage: ynh_clean_check_starting -ynh_clean_check_starting () { - # Stop the execution of tail. - kill -s 15 $pid_tail 2>&1 - ynh_secure_remove "$templog" 2>&1 -} - -# Read the value of a key in a ynh manifest file -# -# usage: ynh_read_manifest manifest key -# | arg: -m, --manifest= - Path of the manifest to read -# | arg: -k, --key= - Name of the key to find -ynh_read_manifest () { - # Declare an array to define the options of this helper. - declare -Ar args_array=( [m]=manifest= [k]=manifest_key= ) - local manifest - local manifest_key - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - if [ ! -e "$manifest" ]; then - # If the manifest isn't found, try the common place for backup and restore script. - manifest="../settings/manifest.json" - fi - - jq ".$manifest_key" "$manifest" --raw-output -} - -# Read the upstream version from the manifest -# The version number in the manifest is defined by ~ynh -# For example : 4.3-2~ynh3 -# This include the number before ~ynh -# In the last example it return 4.3-2 -# -# usage: ynh_app_upstream_version [-m manifest] -# | arg: -m, --manifest= - Path of the manifest to read -ynh_app_upstream_version () { - declare -Ar args_array=( [m]=manifest= ) - local manifest - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - manifest="${manifest:-../manifest.json}" - version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") - echo "${version_key/~ynh*/}" -} - -# Read package version from the manifest -# The version number in the manifest is defined by ~ynh -# For example : 4.3-2~ynh3 -# This include the number after ~ynh -# In the last example it return 3 -# -# usage: ynh_app_package_version [-m manifest] -# | arg: -m, --manifest= - Path of the manifest to read -ynh_app_package_version () { - declare -Ar args_array=( [m]=manifest= ) - local manifest - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - manifest="${manifest:-../manifest.json}" - version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") - echo "${version_key/*~ynh/}" -} - -# Checks the app version to upgrade with the existing app version and returns: -# - UPGRADE_APP if the upstream app version has changed -# - UPGRADE_PACKAGE if only the YunoHost package has changed -# -## It stops the current script without error if the package is up-to-date -# -# This helper should be used to avoid an upgrade of an app, or the upstream part -# of it, when it's not needed -# -# To force an upgrade, even if the package is up to date, -# you have to set the variable YNH_FORCE_UPGRADE before. -# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp -# -# usage: ynh_check_app_version_changed -ynh_check_app_version_changed () { - local force_upgrade=${YNH_FORCE_UPGRADE:-0} - local package_check=${PACKAGE_CHECK_EXEC:-0} - - # By default, upstream app version has changed - local return_value="UPGRADE_APP" - - local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0) - local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")" - local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0) - local update_upstream_version="$(ynh_app_upstream_version)" - - if [ "$current_version" == "$update_version" ] ; then - # Complete versions are the same - if [ "$force_upgrade" != "0" ] - then - echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2 - unset YNH_FORCE_UPGRADE - elif [ "$package_check" != "0" ] - then - echo "Upgrade forced for package check." >&2 - else - ynh_die "Up-to-date, nothing to do" 0 - fi - elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then - # Upstream versions are the same, only YunoHost package versions differ - return_value="UPGRADE_PACKAGE" - fi - echo $return_value -} diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 5ba2946a2..c6603ab38 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -1,109 +1,96 @@ #!/bin/bash -# Extract a key from a plain command output +# Determine whether the scripts is executed from a terminal or the admin JS. # -# example: yunohost user info tata --output-as plain | ynh_get_plain_key mail -# -# usage: ynh_get_plain_key key [subkey [subsubkey ...]] -# | ret: string - the key's value -ynh_get_plain_key() { - local prefix="#" - local founded=0 - local key=$1 - shift - while read line; do - if [[ "$founded" == "1" ]] ; then - [[ "$line" =~ ^${prefix}[^#] ]] && return - echo $line - elif [[ "$line" =~ ^${prefix}${key}$ ]]; then - if [[ -n "${1:-}" ]]; then - prefix+="#" - key=$1 - shift - else - founded=1 - fi - fi - done +# [internal] +ynh_is_term () { + # Return true if $TERM return xterm + # Which means the script is executed from a terminal, not from the admin JS + [[ "$TERM" =~ "xterm" ]] } -# Restore a previous backup if the upgrade process failed +# Manage a fail of the script +# +# [internal] # # usage: -# ynh_backup_before_upgrade +# ynh_exit_properly is used only by the helper ynh_abort_if_errors. +# You should not use it directly. +# Instead, add to your script: # ynh_clean_setup () { -# ynh_restore_upgradebackup +# instructions... # } -# ynh_abort_if_errors # -ynh_restore_upgradebackup () { - echo "Upgrade failed." >&2 - local app_bck=${app//_/-} # Replace all '_' by '-' - - NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} - - if [ "$NO_BACKUP_UPGRADE" -eq 0 ] - then - # Check if an existing backup can be found before removing and restoring the application. - if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$backup_number - then - # Remove the application then restore it - sudo yunohost app remove $app - # Restore the backup - sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug - ynh_die --message="The app was restored to the way it was before the failed upgrade." - fi - else - echo "\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" >&2 - fi -} - -# Make a backup in case of failed upgrade +# This function provide a way to clean some residual of installation that not managed by remove script. # -# usage: -# ynh_backup_before_upgrade -# ynh_clean_setup () { -# ynh_restore_upgradebackup -# } -# ynh_abort_if_errors +# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script # -ynh_backup_before_upgrade () { - if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ] - then - echo "This app doesn't have any backup script." >&2 - return +ynh_exit_properly () { + local exit_code=$? + if [ "$exit_code" -eq 0 ]; then + exit 0 # Exit without error if the script ended correctly fi - backup_number=1 - local old_backup_number=2 - local app_bck=${app//_/-} # Replace all '_' by '-' - NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} - if [ "$NO_BACKUP_UPGRADE" -eq 0 ] - then - # Check if a backup already exists with the prefix 1 - if sudo yunohost backup list | grep -q $app_bck-pre-upgrade1 - then - # Prefix becomes 2 to preserve the previous backup - backup_number=2 - old_backup_number=1 - fi + trap '' EXIT # Ignore new exit signals + set +eu # Do not exit anymore if a command fail or if a variable is empty - # Create backup - sudo BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug - if [ "$?" -eq 0 ] - then - # If the backup succeeded, remove the previous backup - if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$old_backup_number - then - # Remove the previous backup only if it exists - sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null + echo -e "!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" >&2 + + # If the script is executed from a terminal, dump the end of the log that precedes the crash. + if ynh_is_term + then + # Unset xtrace to not spoil the log + set +x + + local ynh_log="/var/log/yunohost/yunohost-cli.log" + + # Wait for the log to be fill with the data until the crash. + local timeout=0 + while ! tail --lines=20 "$ynh_log" | grep --quiet "ynh_exit_properly" + do + ((timeout++)) + if [ $timeout -eq 500 ]; then + break fi - else - ynh_die --message="Backup failed, the upgrade process was aborted." - fi - else - echo "\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" - fi + done + + echo -e "\e[34m\e[1mPlease find here an extract of the log before the crash:\e[0m" >&2 + # Tail the last 30 lines of log of YunoHost + # But remove all lines after "ynh_exit_properly" + # Remove the timestamp at the beginning of the line + # Remove "yunohost.hook..." + # Add DEBUG and color it at the beginning of each log line. + echo -e "$(tail --lines=30 "$ynh_log" \ + | sed '1,/ynh_exit_properly/!d' \ + | sed 's/^[[:digit:]: ,-]*//g' \ + | sed 's/ *yunohost.hook.*\]/ -/g' \ + | sed 's/^WARNING /&/g' \ + | sed 's/^DEBUG /& /g' \ + | sed 's/^INFO /& /g' \ + | sed 's/^/\\e[34m\\e[1m[DEBUG]\\e[0m: /g')" >&2 + set -x + fi + + if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script. + ynh_clean_setup # Call the function to do specific cleaning for the app. + fi + + ynh_die # Exit with error status +} + +# Exits if an error occurs during the execution of the script. +# +# usage: ynh_abort_if_errors +# +# This configure the rest of the script execution such that, if an error occurs +# or if an empty variable is used, the execution of the script stops +# immediately and a call to `ynh_clean_setup` is triggered if it has been +# defined by your script. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_abort_if_errors () { + set -eu # Exit if a command fail, and if a variable is used unset. + trap ynh_exit_properly EXIT # Capturing exit signals on shell script } # Download, check integrity, uncompress and patch the source from app.src @@ -125,7 +112,7 @@ ynh_backup_before_upgrade () { # SOURCE_IN_SUBDIR=false # # (Optionnal) Name of the local archive (offline setup support) # # default: ${src_id}.${src_format} -# SOURCE_FILENAME=example.tar.gz +# SOURCE_FILENAME=example.tar.gz # # (Optional) If it set as false don't extract the source. # # (Useful to get a debian package or a python wheel.) # # default: true @@ -150,6 +137,8 @@ ynh_backup_before_upgrade () { # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] # | arg: -d, --dest_dir - Directory where to setup sources # | arg: -s, --source_id - Name of the app, if the package contains more than one app +# +# Requires YunoHost version 2.6.4 or higher. ynh_setup_source () { # Declare an array to define the options of this helper. local legacy_args=ds @@ -160,15 +149,22 @@ ynh_setup_source () { ynh_handle_getopts_args "$@" source_id="${source_id:-app}" # If the argument is not given, source_id equals "app" + local src_file_path="$YNH_CWD/../conf/${source_id}.src" + # In case of restore script the src file is in an other path. + # So try to use the restore path if the general path point to no file. + if [ ! -e "$src_file_path" ]; then + src_file_path="$YNH_CWD/../settings/conf/${source_id}.src" + fi + # Load value from configuration file (see above for a small doc about this file # format) - local src_url=$(grep 'SOURCE_URL=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_sum=$(grep 'SOURCE_SUM=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_format=$(grep 'SOURCE_FORMAT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_extract=$(grep 'SOURCE_EXTRACT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_filename=$(grep 'SOURCE_FILENAME=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut -d= -f2-) + local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut -d= -f2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut -d= -f2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut -d= -f2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut -d= -f2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut -d= -f2-) + local src_filename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut -d= -f2-) # Default value src_sumprg=${src_sumprg:-sha256sum} @@ -199,7 +195,7 @@ ynh_setup_source () { then mv $src_filename $dest_dir elif [ "$src_format" = "zip" ] - then + then # Zip format # Using of a temp directory, because unzip doesn't manage --strip-components if $src_in_subdir ; then @@ -249,45 +245,47 @@ ynh_setup_source () { # $domain and $path_url should be defined externally (and correspond to the domain.tld and the /path (of the app?)) # # example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2" -# +# # usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ... # | arg: page_uri - Path (relative to $path_url) of the page where POST data will be sent # | arg: key1=value1 - (Optionnal) POST key and corresponding value # | arg: key2=value2 - (Optionnal) Another POST key and corresponding value # | arg: ... - (Optionnal) More POST keys and values +# +# Requires YunoHost version 2.6.4 or higher. ynh_local_curl () { - # Define url of page to curl - local local_page=$(ynh_normalize_url_path $1) - local full_path=$path_url$local_page - - if [ "${path_url}" == "/" ]; then - full_path=$local_page - fi - - local full_page_url=https://localhost$full_path + # Define url of page to curl + local local_page=$(ynh_normalize_url_path $1) + local full_path=$path_url$local_page - # Concatenate all other arguments with '&' to prepare POST data - local POST_data="" - local arg="" - for arg in "${@:2}" - do - POST_data="${POST_data}${arg}&" - done - if [ -n "$POST_data" ] - then - # Add --data arg and remove the last character, which is an unecessary '&' - POST_data="--data ${POST_data::-1}" - fi - - # Wait untils nginx has fully reloaded (avoid curl fail with http2) - sleep 2 + if [ "${path_url}" == "/" ]; then + full_path=$local_page + fi - # Curl the URL - curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" + local full_page_url=https://localhost$full_path + + # Concatenate all other arguments with '&' to prepare POST data + local POST_data="" + local arg="" + for arg in "${@:2}" + do + POST_data="${POST_data}${arg}&" + done + if [ -n "$POST_data" ] + then + # Add --data arg and remove the last character, which is an unecessary '&' + POST_data="--data ${POST_data::-1}" + fi + + # Wait untils nginx has fully reloaded (avoid curl fail with http2) + sleep 2 + + # Curl the URL + curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" } # Render templates with Jinja2 -# +# # Attention : Variables should be exported before calling this helper to be # accessible inside templates. # @@ -303,3 +301,220 @@ ynh_render_template() { jinja2.Template(sys.stdin.read() ).render(os.environ));' < $template_path > $output_path } + +# Fetch the Debian release codename +# +# usage: ynh_get_debian_release +# | ret: The Debian release codename (i.e. jessie, stretch, ...) +# +# Requires YunoHost version 2.7.12 or higher. +ynh_get_debian_release () { + echo $(lsb_release --codename --short) +} + +# Create a directory under /tmp +# +# [internal] +# +# Deprecated helper +# +# usage: ynh_mkdir_tmp +# | ret: the created directory path +ynh_mkdir_tmp() { + ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated." + ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \ +properly with chmod/chown." + local TMP_DIR=$(mktemp -d) + + # Give rights to other users could be a security risk. + # But for retrocompatibility we need it. (This helpers is deprecated) + chmod 755 $TMP_DIR + echo $TMP_DIR +} + +# Remove a file or a directory securely +# +# usage: ynh_secure_remove --file=path_to_remove +# | arg: -f, --file - File or directory to remove +# +# Requires YunoHost version 2.6.4 or higher. +ynh_secure_remove () { + # Declare an array to define the options of this helper. + local legacy_args=f + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local forbidden_path=" \ + /var/www \ + /home/yunohost.app" + + if [ $# -ge 2 ] + then + ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." + fi + + if [[ "$forbidden_path" =~ "$file" \ + # Match all paths or subpaths in $forbidden_path + || "$file" =~ ^/[[:alnum:]]+$ \ + # Match all first level paths from / (Like /var, /root, etc...) + || "${file:${#file}-1}" = "/" ]] + # Match if the path finishes by /. Because it seems there is an empty variable + then + ynh_print_warn --message="Avoid deleting $file." + else + if [ -e "$file" ] + then + sudo rm -R "$file" + else + ynh_print_info --message="$file wasn't deleted because it doesn't exist." + fi + fi +} + +# Extract a key from a plain command output +# +# example: yunohost user info tata --output-as plain | ynh_get_plain_key mail +# +# usage: ynh_get_plain_key key [subkey [subsubkey ...]] +# | ret: string - the key's value +# +# Requires YunoHost version 2.2.4 or higher. +ynh_get_plain_key() { + local prefix="#" + local founded=0 + local key=$1 + shift + while read line; do + if [[ "$founded" == "1" ]] ; then + [[ "$line" =~ ^${prefix}[^#] ]] && return + echo $line + elif [[ "$line" =~ ^${prefix}${key}$ ]]; then + if [[ -n "${1:-}" ]]; then + prefix+="#" + key=$1 + shift + else + founded=1 + fi + fi + done +} + +# Read the value of a key in a ynh manifest file +# +# usage: ynh_read_manifest manifest key +# | arg: -m, --manifest= - Path of the manifest to read +# | arg: -k, --key= - Name of the key to find +# +# Requires YunoHost version 3.?.? or higher. +ynh_read_manifest () { + # Declare an array to define the options of this helper. + local legacy_args=mk + declare -Ar args_array=( [m]=manifest= [k]=manifest_key= ) + local manifest + local manifest_key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [ ! -e "$manifest" ]; then + # If the manifest isn't found, try the common place for backup and restore script. + manifest="../settings/manifest.json" + fi + + jq ".$manifest_key" "$manifest" --raw-output +} + +# Read the upstream version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number before ~ynh +# In the last example it return 4.3-2 +# +# usage: ynh_app_upstream_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +# +# Requires YunoHost version 3.?.? or higher. +ynh_app_upstream_version () { + # Declare an array to define the options of this helper. + local legacy_args=m + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + echo "${version_key/~ynh*/}" +} + +# Read package version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number after ~ynh +# In the last example it return 3 +# +# usage: ynh_app_package_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +# +# Requires YunoHost version 3.?.? or higher. +ynh_app_package_version () { + # Declare an array to define the options of this helper. + local legacy_args=m + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + echo "${version_key/*~ynh/}" +} + +# Checks the app version to upgrade with the existing app version and returns: +# - UPGRADE_APP if the upstream app version has changed +# - UPGRADE_PACKAGE if only the YunoHost package has changed +# +# It stops the current script without error if the package is up-to-date +# +# This helper should be used to avoid an upgrade of an app, or the upstream part +# of it, when it's not needed +# +# To force an upgrade, even if the package is up to date, +# you have to set the variable YNH_FORCE_UPGRADE before. +# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp +# +# usage: ynh_check_app_version_changed +# +# Requires YunoHost version 3.?.? or higher. +ynh_check_app_version_changed () { + local force_upgrade=${YNH_FORCE_UPGRADE:-0} + local package_check=${PACKAGE_CHECK_EXEC:-0} + + # By default, upstream app version has changed + local return_value="UPGRADE_APP" + + local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0) + local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")" + local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0) + local update_upstream_version="$(ynh_app_upstream_version)" + + if [ "$current_version" == "$update_version" ] ; then + # Complete versions are the same + if [ "$force_upgrade" != "0" ] + then + ynh_print_info --message="Upgrade forced by YNH_FORCE_UPGRADE." + unset YNH_FORCE_UPGRADE + elif [ "$package_check" != "0" ] + then + ynh_print_info --message="Upgrade forced for package check." + else + ynh_die "Up-to-date, nothing to do" 0 + fi + elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then + # Upstream versions are the same, only YunoHost package versions differ + return_value="UPGRADE_PACKAGE" + fi + echo $return_value +} From a4f3248545337ad2e45f3ff590e67a54e80605da Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 8 May 2019 01:04:57 +0200 Subject: [PATCH 4/7] Restore usage of ynh_print_err --- data/helpers.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index c6603ab38..d24474ebc 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -34,7 +34,7 @@ ynh_exit_properly () { trap '' EXIT # Ignore new exit signals set +eu # Do not exit anymore if a command fail or if a variable is empty - echo -e "!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" >&2 + ynh_print_err --message="!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" # If the script is executed from a terminal, dump the end of the log that precedes the crash. if ynh_is_term From 95d2c8f79940feb8bf757bec48422bcc484b9253 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 May 2019 18:31:02 +0200 Subject: [PATCH 5/7] Inject a new environment variable YNH_INTERFACE to test inside helpers if we're using Yunohost from the CLI or API --- data/helpers.d/utils | 13 ++----------- src/yunohost/hook.py | 4 +++- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index d24474ebc..a561b2f73 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -1,14 +1,5 @@ #!/bin/bash -# Determine whether the scripts is executed from a terminal or the admin JS. -# -# [internal] -ynh_is_term () { - # Return true if $TERM return xterm - # Which means the script is executed from a terminal, not from the admin JS - [[ "$TERM" =~ "xterm" ]] -} - # Manage a fail of the script # # [internal] @@ -36,8 +27,8 @@ ynh_exit_properly () { ynh_print_err --message="!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" - # If the script is executed from a terminal, dump the end of the log that precedes the crash. - if ynh_is_term + # If the script is executed from the CLI, dump the end of the log that precedes the crash. + if [ "$YNH_INTERFACE" == "cli" ] then # Unset xtrace to not spoil the log set +x diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 2841dd425..42807fdf7 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -28,7 +28,7 @@ import re import tempfile from glob import iglob -from moulinette import m18n +from moulinette import m18n, msettings from yunohost.utils.error import YunohostError from moulinette.utils import log from moulinette.utils.filesystem import read_json @@ -337,6 +337,8 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, env = {} env['YNH_CWD'] = chdir + env['YNH_INTERFACE'] = msettings.get('interface') + stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo") env['YNH_STDINFO'] = stdinfo From 899d3d4f105e1627a0c10bd939ecca74035f9365 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 May 2019 18:46:46 +0200 Subject: [PATCH 6/7] Improve ynh_exit_properly detection for case where failure happens right at the beginning --- data/helpers.d/utils | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index a561b2f73..73638dc90 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -37,7 +37,7 @@ ynh_exit_properly () { # Wait for the log to be fill with the data until the crash. local timeout=0 - while ! tail --lines=20 "$ynh_log" | grep --quiet "ynh_exit_properly" + while ! tail --lines=20 "$ynh_log" | grep --quiet "+ ynh_exit_properly" do ((timeout++)) if [ $timeout -eq 500 ]; then @@ -52,7 +52,7 @@ ynh_exit_properly () { # Remove "yunohost.hook..." # Add DEBUG and color it at the beginning of each log line. echo -e "$(tail --lines=30 "$ynh_log" \ - | sed '1,/ynh_exit_properly/!d' \ + | sed '1,/+ ynh_exit_properly/!d' \ | sed 's/^[[:digit:]: ,-]*//g' \ | sed 's/ *yunohost.hook.*\]/ -/g' \ | sed 's/^WARNING /&/g' \ From 6376b05b604415de5f0236d0b05e144a09cf21e7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 May 2019 19:29:18 +0200 Subject: [PATCH 7/7] Add a small tempo to avoid the next message being mixed up with other DEBUG messages --- data/helpers.d/utils | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 73638dc90..8b6c11a80 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -25,6 +25,9 @@ ynh_exit_properly () { trap '' EXIT # Ignore new exit signals set +eu # Do not exit anymore if a command fail or if a variable is empty + # Small tempo to avoid the next message being mixed up with other DEBUG messages + sleep 0.5 + ynh_print_err --message="!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" # If the script is executed from the CLI, dump the end of the log that precedes the crash.