YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

logs: misc ad-hoc tweaks to limit the noise in log sharing

Browse source
This commit is contained in:
Alexandre Aubin 2024-06-21 14:39:49 +02:00
parent a25033bba5
commit 06c8fbc881
8 changed files with 77 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
helpers/helpers.v2.1.d/apt
View file

@ -16,15 +16,14 @@ YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true"
#
# Requires YunoHost version 2.6.4 or higher.
ynh_apt_install_dependencies() {
local dependencies=$@
# Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below)
dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')"
local dependencies=${dependencies//|/ | }
# Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below)
local dependencies="$(sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g' <<< "$@" | sed 's/|/ | /')"
local version=$(ynh_read_manifest "version")
local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps
# Handle specific versions
if [[ "$dependencies" =~ [\<=\>] ]]; then
if grep '[<=>]' <<< "$dependencies"; then
# Replace version specifications by relationships syntax
# https://www.debian.org/doc/debian-policy/ch-relationships.html
# Sed clarification
@ -33,7 +32,7 @@ ynh_apt_install_dependencies() {
# \+ matches one or more occurence of the previous characters, for >= or >>.
# [^,]\+ matches all characters except ','
# Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)'
dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')"
dependencies="$(sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g' <<< "$dependencies")"
fi
# ############################## #
@ -43,7 +42,7 @@ ynh_apt_install_dependencies() {
# Check for specific php dependencies which requires sury
# This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni"
# The (?<=php) syntax corresponds to lookbehind ;)
local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u)
local specific_php_version=$(grep -oP '(?<=php)[0-9.]+(?=-|\>|)' <<< "$dependencies" | sort -u)
if [[ -n "$specific_php_version" ]]
then
@ -128,21 +127,22 @@ EOF
# NB: this is in a subshell (though not sure why exactly not just use pushd/popd...)
cd "$TMPDIR"
# Install the fake package without its dependencies with dpkg --force-depends
LC_ALL=C equivs-build ./control 2>&1
LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" 2>&1 | tee ./dpkg_log
LC_ALL=C equivs-build ./control > ./equivs_log 2>&1 || { cat ./equivs_log; false; }
LC_ALL=C dpkg --force-depends --install "./${app_ynh_deps}_${version}_all.deb" > ./dpkg_log 2>&1
)
# Then install the missing dependencies with apt install
_ynh_apt_install --fix-broken \
|| { # If the installation failed
# (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
# Parse the list of problematic dependencies from dpkg's log ...
# (relevant lines look like: "foo-ynh-deps depends on bar; however:")
local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')"
# Fake an install of those dependencies to see the errors
# The sed command here is, Print only from 'Reading state info' to the end.
[[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
ynh_die "Unable to install dependencies"
_ynh_apt_install --fix-broken || {
# If the installation failed
# (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
# Parse the list of problematic dependencies from dpkg's log ...
# (relevant lines look like: "foo-ynh-deps depends on bar; however:")
cat $TMPDIR/dpkg_log
local problematic_dependencies="$(grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' $TMPDIR/dpkg_log | tr '\n' ' ')"
# Fake an install of those dependencies to see the errors
# The sed command here is, Print only from 'Reading state info' to the end.
[[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
ynh_die "Unable to install dependencies"
}
rm --recursive --force "$TMPDIR" # Remove the temp dir.

4
helpers/helpers.v2.1.d/backup
View file

@ -211,6 +211,7 @@ _ynh_file_checksum_exists() {
#
# usage: ynh_store_file_checksum /path/to/file
ynh_store_file_checksum() {
set +o xtrace # set +x
local file=$1
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
@ -231,6 +232,7 @@ ynh_store_file_checksum() {
fi
# Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
unset backup_file_checksum
set -o xtrace # set -x
}
# Verify the checksum and backup the file if it's different
@ -240,6 +242,7 @@ ynh_store_file_checksum() {
# This helper is primarily meant to allow to easily backup personalised/manually
# modified config files.
ynh_backup_if_checksum_is_different() {
set +o xtrace # set +x
local file=$1
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name)
@ -263,6 +266,7 @@ ynh_backup_if_checksum_is_different() {
fi
fi
fi
set -o xtrace # set -x
}
# Delete a file checksum from the app settings

7
helpers/helpers.v2.1.d/getopts
View file

@ -46,10 +46,13 @@
#
# Requires YunoHost version 3.2.2 or higher.
ynh_handle_getopts_args() {
# Trick to only re-enable debugging if it was set before
local xtrace_enable=$(set +o | grep xtrace)
# Manage arguments only if there's some provided
set +o xtrace # set +x
if [ $# -eq 0 ]; then
set -o xtrace # set -x
eval "$xtrace_enable"
return
fi
@ -181,5 +184,5 @@ ynh_handle_getopts_args() {
# Call parse_arg and pass the modified list of args as an array of arguments.
parse_arg "${arguments[@]}"
set -o xtrace # set -x
eval "$xtrace_enable"
}

40
helpers/helpers.v2.1.d/php
View file

@ -176,6 +176,8 @@ ynh_config_remove_phpfpm() {
#
_ynh_get_scalable_phpfpm() {
set +o xtrace # set +x
# If no usage provided, default to the value existing in setting ... or to low
local fpm_usage_in_setting=$(ynh_app_setting_get --key=fpm_usage)
local usage=${fpm_usage_in_setting:-low}
@ -195,16 +197,6 @@ _ynh_get_scalable_phpfpm() {
footprint=50
fi
# Define the factor to determine min_spare_servers
# to avoid having too few children ready to start for heavy apps
if [ $footprint -le 20 ]; then
min_spare_servers_factor=8
elif [ $footprint -le 35 ]; then
min_spare_servers_factor=5
else
min_spare_servers_factor=3
fi
# Define the way the process manager handle child processes.
if [ "$usage" = "low" ]; then
php_pm=ondemand
@ -216,9 +208,6 @@ _ynh_get_scalable_phpfpm() {
ynh_die "Does not recognize '$usage' as an usage value."
fi
# Get the total of RAM available, except swap.
local max_ram=$(ynh_get_ram --total)
at_least_one() {
# Do not allow value below 1
if [ $1 -le 0 ]; then
@ -228,10 +217,13 @@ _ynh_get_scalable_phpfpm() {
fi
}
# Get the total of RAM available, except swap.
local total_ram=$(ynh_get_ram --total)
# Define pm.max_children
# The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app.
# So if PHP-FPM start the maximum of children, it won't exceed half of the ram.
php_max_children=$(($max_ram / 2 / $footprint))
php_max_children=$(($total_ram / 2 / $footprint))
# If process manager is set as static, use half less children.
# Used as static, there's always as many children as the value of pm.max_children
if [ "$php_pm" = "static" ]; then
@ -253,6 +245,17 @@ _ynh_get_scalable_phpfpm() {
fi
if [ "$php_pm" = "dynamic" ]; then
# Define the factor to determine min_spare_servers
# to avoid having too few children ready to start for heavy apps
if [ $footprint -le 20 ]; then
min_spare_servers_factor=8
elif [ $footprint -le 35 ]; then
min_spare_servers_factor=5
else
min_spare_servers_factor=3
fi
# Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager
php_min_spare_servers=$(($php_max_children / $min_spare_servers_factor))
php_min_spare_servers=$(at_least_one $php_min_spare_servers)
@ -267,6 +270,15 @@ _ynh_get_scalable_phpfpm() {
php_max_spare_servers=0
php_start_servers=0
fi
set -o xtrace # set -x
# For debugging, since otherwise things are hidden with set +x/-x
echo "php_pm: $php_pm"
echo "php_max_children: $php_max_children"
echo "php_min_spare_servers: $php_min_spare_servers"
echo "php_max_spare_servers: $php_max_spare_servers"
echo "php_start_servers: $php_start_servers"
}
# Execute a command with Composer

4
helpers/helpers.v2.1.d/setting
View file

@ -68,6 +68,8 @@ ynh_app_setting_delete() {
# [internal]
#
ynh_app_setting() {
# Trick to only re-enable debugging if it was set before
local xtrace_enable=$(set +o | grep xtrace)
set +o xtrace # set +x
ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <<EOF
import os, yaml, sys
@ -91,5 +93,5 @@ else:
with open(setting_file, "w") as f:
yaml.safe_dump(settings, f, default_flow_style=False)
EOF
set -o xtrace # set -x
eval "$xtrace_enable"
}

4
helpers/helpers.v2.1.d/systemd
View file

@ -114,7 +114,7 @@ ynh_systemctl() {
# Start the timeout and try to find wait_until
if [[ -n "${wait_until:-}" ]]; then
set +x
set +o xtrace # set +x
local i=0
local starttime=$(date +%s)
for i in $(seq 1 $timeout); do
@ -144,7 +144,7 @@ ynh_systemctl() {
fi
sleep 1
done
set -x
set -o xtrace # set -x
if [ $i -ge 3 ]; then
echo "" >&2
fi

7
helpers/helpers.v2.1.d/utils
View file

@ -158,7 +158,6 @@ ynh_setup_source() {
local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')"
local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')"
local src_sumprg="sha256sum"
local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')"
local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')"
src_in_subdir=${src_in_subdir:-true}
@ -215,7 +214,7 @@ ynh_setup_source() {
[ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?"
# If the file was prefetched but somehow doesn't match the sum, rm and redownload it
if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | sha256sum --check --status
then
rm -f "$src_filename"
fi
@ -233,9 +232,9 @@ ynh_setup_source() {
fi
# Check the control sum
if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status
if ! echo "${src_sum} ${src_filename}" | sha256sum --check --status
then
local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)"
local actual_sum="$(sha256sum ${src_filename} | cut --delimiter=' ' --fields=1)"
local actual_size="$(du -hs ${src_filename} | cut --fields=1)"
rm -f ${src_filename}
ynh_die "Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})."

15
src/log.py
View file

@ -45,12 +45,18 @@ LOG_FILE_EXT = ".log"
BORING_LOG_LINES = [
r"set [+-]x$",
r"set [+-]o xtrace$",
r"\+ set \+o$",
r"\+ grep xtrace$",
r"local 'xtrace_enable=",
r"set [+-]o errexit$",
r"set [+-]o nounset$",
r"trap '' EXIT",
r"local \w+$",
r"local exit_code=(1|0)$",
r"local legacy_args=.*$",
r"local _globalapp=.*$",
r"local checksum_setting_name=.*$",
r"ynh_app_setting ", # (note the trailing space to match the "low level" one called by other setting helpers)
r"local -A args_array$",
r"args_array=.*$",
r"ret_code=1",
@ -62,8 +68,17 @@ BORING_LOG_LINES = [
r"\[?\['? -n '' '?\]\]?$",
r"rm -rf /var/cache/yunohost/download/$",
r"type -t ynh_clean_setup$",
r"DEBUG - \+ unset \S+$",
r"DEBUG - \+ echo '",
r"DEBUG - \+ LC_ALL=C$",
r"DEBUG - \+ DEBIAN_FRONTEND=noninteractive$",
r"DEBUG - \+ exit (1|0)$",
r"DEBUG - \+ app=\S+$",
r"DEBUG - \+\+ app=\S+$",
r"DEBUG - \+\+ jq -r .\S+$",
r"DEBUG - \+\+ sed 's/\^null\$//'$",
"DEBUG - \\+ sed --in-place \$'s\\\\001",
"DEBUG - \\+ sed --in-place 's\u0001.*$",
]