YunoHost/package_check
1
0
Fork 0
mirror of https://github.com/YunoHost/package_check.git synced 2024-09-03 20:06:20 +02:00
Code Issues Projects Releases Packages Wiki Activity

Super rework/cleanup of the logging and other misc cleanup...

Browse source
This commit is contained in:
Alexandre Aubin 2020-11-30 03:45:51 +01:00
parent 08db4b90eb
commit d6345d530a
6 changed files with 2745 additions and 2965 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

183
package_check.sh
View file

@ -20,8 +20,6 @@ clean_exit () {
LXC_TURNOFF LXC_TURNOFF
# Remove temporary files # Remove temporary files
rm -f "$temp_log"
rm -f "$temp_result"
rm -f "$script_dir/url_output" rm -f "$script_dir/url_output"
rm -f "$script_dir/curl_print" rm -f "$script_dir/curl_print"
rm -f "$script_dir/manifest_extract" rm -f "$script_dir/manifest_extract"
@ -178,11 +176,11 @@ package_check.sh [OPTION]... PACKAGE_TO_CHECK
-l, --build-lxc -l, --build-lxc
Install LXC and build the container if necessary. Install LXC and build the container if necessary.
-y, --bash-mode -y, --bash-mode
Do not ask for continue check. Ignore auto_remove. Do not ask for continue check. Ignore auto_remove.
-r, --show-resources -r, --show-resources
Show the unavailable resources when accessing the url. Show the unavailable resources when accessing the url.
EOF EOF
exit 0 exit 0
fi fi
#================================================= #=================================================
@ -264,18 +262,18 @@ rm "$script_dir/pcheck.lock"
exec "$script_dir/package_check.sh" "${arguments[@]}" exec "$script_dir/package_check.sh" "${arguments[@]}"
EOF EOF
# Give the execution right # Give the execution right
chmod +x "$script_dir/upgrade_script.sh" chmod +x "$script_dir/upgrade_script.sh"
# Temporary upgrade fix # Temporary upgrade fix
# Check if lynx is already installed. # Check if lynx is already installed.
if [ ! -e "$(which lynx)" ] if [ ! -e "$(which lynx)" ]
then then
sudo apt-get install -y lynx sudo apt-get install -y lynx
fi fi
# Start the upgrade script by replacement of this process # Start the upgrade script by replacement of this process
exec "$script_dir/upgrade_script.sh" exec "$script_dir/upgrade_script.sh"
fi fi
fi fi
@ -409,12 +407,6 @@ fi
# Complete result log. Complete log of YunoHost # Complete result log. Complete log of YunoHost
complete_log="$script_dir/Complete.log" complete_log="$script_dir/Complete.log"
# Partial YunoHost log, just the log for the current test
temp_log="$script_dir/temp_yunohost-cli.log"
# Temporary result log
temp_result="$script_dir/temp_result.log"
# Result log with warning and error only
test_result="$script_dir/Test_results.log"
# Real YunoHost log # Real YunoHost log
yunohost_log="/var/lib/lxc/$lxc_name/rootfs/var/log/yunohost/yunohost-cli.log" yunohost_log="/var/lib/lxc/$lxc_name/rootfs/var/log/yunohost/yunohost-cli.log"
@ -425,9 +417,9 @@ test_user=package_checker
# Load all functions # Load all functions
#================================================= #=================================================
source "$script_dir/sub_scripts/common.sh"
source "$script_dir/sub_scripts/launcher.sh" source "$script_dir/sub_scripts/launcher.sh"
source "$script_dir/sub_scripts/testing_process.sh" source "$script_dir/sub_scripts/testing_process.sh"
source "$script_dir/sub_scripts/log_extractor.sh"
#================================================= #=================================================
# Check LXC # Check LXC
@ -444,8 +436,8 @@ then
# If lxc's not installed and build_lxc set. Asks to build the container. # If lxc's not installed and build_lxc set. Asks to build the container.
build_lxc=2 build_lxc=2
else else
ECHO_FORMAT "LXC is not installed or the container $lxc_name doesn't exist.\n" "red" critical "LXC is not installed or the container $lxc_name doesn't exist."
ECHO_FORMAT "Use the script 'lxc_build.sh' to fix them.\n" "red" critical "Use the script 'lxc_build.sh' to fix them."
clean_exit 1 clean_exit 1
fi fi
fi fi
@ -496,13 +488,11 @@ then
app_arg="${app_arg%%/tree/*}" app_arg="${app_arg%%/tree/*}"
fi fi
ECHO_FORMAT "Test the package $app_arg" "blue" "bold"
if [ -n "$gitbranch" ] if [ -n "$gitbranch" ]
then then
ECHO_FORMAT " on the branch ${gitbranch##-b }.\n" "blue" "bold" branch_msg=" on the branch ${gitbranch##-b }"
else
ECHO_FORMAT ".\n" "blue" "bold"
fi fi
info "Test the package $app_arg $branch_msg"
# Remove the previous package if it's still here. # Remove the previous package if it's still here.
rm -rf "$script_dir"/*_check rm -rf "$script_dir"/*_check
@ -524,7 +514,7 @@ then
then then
gitbranch="-b stable" gitbranch="-b stable"
else else
ECHO_FORMAT "Unable to find a default branch to test (master or stable)" "red" critical "Unable to find a default branch to test (master or stable)"
clean_exit 1 clean_exit 1
fi fi
fi fi
@ -532,7 +522,7 @@ then
# Clone the repository # Clone the repository
git clone $app_arg $gitbranch "$package_path" git clone $app_arg $gitbranch "$package_path"
# If it's a local directory # If it's a local directory
else else
# Do a copy in the directory of Package check # Do a copy in the directory of Package check
cp -a "$app_arg" "$package_path" cp -a "$app_arg" "$package_path"
@ -540,7 +530,7 @@ fi
# Check if the package directory is really here. # Check if the package directory is really here.
if [ ! -d "$package_path" ]; then if [ ! -d "$package_path" ]; then
ECHO_FORMAT "Unable to find the directory $package_path for the package...\n" "red" critical "Unable to find the directory $package_path for the package..."
clean_exit 1 clean_exit 1
fi fi
@ -555,25 +545,18 @@ TEST_RESULTS () {
# Print the test result # Print the test result
print_result () { print_result () {
# Print the name of the test
ECHO_FORMAT "$1: "
# Complete with spaces according to the lenght of the previous string. To align the results
local i=0
for i in `seq ${#1} 30`
do
echo -n " "
done
# Print the result of this test # Print the result of this test
# we use printf to force the length to 30 (filled with space)
testname=$(printf %-30.30s "$1:")
if [ $2 -eq 1 ] if [ $2 -eq 1 ]
then then
ECHO_FORMAT "SUCCESS\n" "lgreen" echo "$testname ${BOLD}${GREEN}SUCCESS${NORMAL}"
elif [ $2 -eq -1 ] elif [ $2 -eq -1 ]
then then
ECHO_FORMAT "FAIL\n" "red" echo "$testname ${BOLD}${RED}FAIL${NORMAL}"
else else
ECHO_FORMAT "Not evaluated.\n" "white" echo "$testname Not evaluated."
fi fi
} }
@ -641,12 +624,12 @@ TEST_RESULTS () {
then level[1]=2 then level[1]=2
else level[1]=0 else level[1]=0
fi fi
fi fi
# Evaluate the second level # Evaluate the second level
# -> The package can be install and remove in all tested configurations. # -> The package can be install and remove in all tested configurations.
if level_can_change 2 if level_can_change 2
then then
# Validated if none install failed # Validated if none install failed
if [ $RESULT_check_sub_dir -ne -1 ] && \ if [ $RESULT_check_sub_dir -ne -1 ] && \
[ $RESULT_check_remove_sub_dir -ne -1 ] && \ [ $RESULT_check_remove_sub_dir -ne -1 ] && \
@ -655,9 +638,9 @@ TEST_RESULTS () {
[ $RESULT_check_private -ne -1 ] && \ [ $RESULT_check_private -ne -1 ] && \
[ $RESULT_check_public -ne -1 ] && \ [ $RESULT_check_public -ne -1 ] && \
[ $RESULT_check_multi_instance -ne -1 ] [ $RESULT_check_multi_instance -ne -1 ]
then level[2]=2 then level[2]=2
else level[2]=0 else level[2]=0
fi fi
fi fi
# Evaluate the third level # Evaluate the third level
@ -671,21 +654,21 @@ TEST_RESULTS () {
then level[3]=2 then level[3]=2
else level[3]=0 else level[3]=0
fi fi
fi fi
# Evaluate the fourth level # Evaluate the fourth level
# -> The package can be backup and restore without error # -> The package can be backup and restore without error
if level_can_change 4 if level_can_change 4
then then
# Validated if backup and restore are ok. Or if backup and restore have been not tested but already validated before. # Validated if backup and restore are ok. Or if backup and restore have been not tested but already validated before.
if ( [ $RESULT_check_backup -eq 1 ] && \ if ( [ $RESULT_check_backup -eq 1 ] && \
[ $RESULT_check_restore -eq 1 ] ) || \ [ $RESULT_check_restore -eq 1 ] ) || \
( [ $RESULT_check_backup -ne -1 ] && \ ( [ $RESULT_check_backup -ne -1 ] && \
[ $RESULT_check_restore -ne -1 ] && \ [ $RESULT_check_restore -ne -1 ] && \
[ "${level[4]}" == "2" ] ) [ "${level[4]}" == "2" ] )
then level[4]=2 then level[4]=2
else level[4]=0 else level[4]=0
fi fi
fi fi
# Evaluate the fifth level # Evaluate the fifth level
@ -704,23 +687,23 @@ TEST_RESULTS () {
then then
level[5]=0 level[5]=0
fi fi
fi fi
# Evaluate the sixth level # Evaluate the sixth level
# -> The package can be backup and restore without error # -> The package can be backup and restore without error
if level_can_change 6 if level_can_change 6
then then
# This is from the linter, tests if app is the Yunohost-apps organization # This is from the linter, tests if app is the Yunohost-apps organization
if [ $RESULT_linter_level_6 -eq 1 ] if [ $RESULT_linter_level_6 -eq 1 ]
then level[6]=2 then level[6]=2
else level[6]=0 else level[6]=0
fi fi
fi fi
# Evaluate the seventh level # Evaluate the seventh level
# -> None errors in all tests performed # -> None errors in all tests performed
if level_can_change 7 if level_can_change 7
then then
level[7]=0 level[7]=0
# Validated if none errors is happened. # Validated if none errors is happened.
if [ $RESULT_global_setup -ne -1 ] && \ if [ $RESULT_global_setup -ne -1 ] && \
@ -739,9 +722,9 @@ TEST_RESULTS () {
[ $RESULT_change_url -ne -1 ] && \ [ $RESULT_change_url -ne -1 ] && \
[ $RESULT_action_config_panel -ne -1 ] && \ [ $RESULT_action_config_panel -ne -1 ] && \
[ $RESULT_linter_level_7 -ge 1 ] [ $RESULT_linter_level_7 -ge 1 ]
then then
level[7]=2 level[7]=2
fi fi
fi fi
# Evaluate the eighth level # Evaluate the eighth level
@ -809,44 +792,38 @@ TEST_RESULTS () {
# If some witness files was missing, it's a big error ! So, the level fall immediately at 0. # If some witness files was missing, it's a big error ! So, the level fall immediately at 0.
if [ $RESULT_witness -eq 1 ] if [ $RESULT_witness -eq 1 ]
then then
ECHO_FORMAT "Some witness files has been deleted during those tests ! It's a very bad thing !\n" "red" "bold" error "Some witness files has been deleted during those tests ! It's a very bad thing !"
global_level=0 global_level=0
fi fi
# If the package linter returned a critical error, the app is flagged as broken / level 0 # If the package linter returned a critical error, the app is flagged as broken / level 0
if [ $RESULT_linter -le -2 ] if [ $RESULT_linter -le -2 ]
then then
ECHO_FORMAT "The package linter reported a critical failure ! App is considered broken !\n" "red" "bold" error "The package linter reported a critical failure ! App is considered broken !"
global_level=0 global_level=0
fi fi
if [ $RESULT_alias_traversal -eq 1 ] if [ $RESULT_alias_traversal -eq 1 ]
then then
ECHO_FORMAT "Issue alias_traversal was detected ! Please see here https://github.com/YunoHost/example_ynh/pull/45 to fix that.\n" "red" "bold" error "Issue alias_traversal was detected ! Please see here https://github.com/YunoHost/example_ynh/pull/45 to fix that."
fi
if [ $RESULT_template -eq 1 ]
then
ECHO_FORMAT "This package does not fully respect the template.\nYou should consider upgrade your package to respect the template of the example app.\n" "red" "bold"
fi fi
# Then, print the levels # Then, print the levels
# Print the global level # Print the global level
verbose_level=$(grep "^$global_level " "$script_dir/levels.list" | cut -c4-) verbose_level=$(grep "^$global_level " "$script_dir/levels.list" | cut -c4-)
ECHO_FORMAT "Level of this application: $global_level ($verbose_level)\n" "white" "bold" info "Level of this application: $global_level ($verbose_level)"
# And print the value for each level # And print the value for each level
for i in `seq 1 10` for i in `seq 1 10`
do do
ECHO_FORMAT "\t Level $i: " display="0"
if [ "${level[$i]}" == "na" ]; then if [ "${level[$i]}" == "na" ]; then
ECHO_FORMAT "N/A\n" display="N/A"
elif [ "${level[$i]}" -ge 1 ]; then elif [ "${level[$i]}" -ge 1 ]; then
ECHO_FORMAT "1\n" "white" "bold" display="1"
else
ECHO_FORMAT "0\n"
fi fi
echo -e "\t Level $i: $display"
done done
} }
@ -861,8 +838,8 @@ check_process="$package_path/check_process"
if [ ! -e "$check_process" ] if [ ! -e "$check_process" ]
then then
ECHO_FORMAT "\nUnable to find a check_process file.\n" "red" error "Unable to find a check_process file."
ECHO_FORMAT "Package check will be used in lower mode.\n" "lyellow" warning "Package check will attempt to automatically guess what tests to run."
check_file=0 check_file=0
fi fi
@ -881,7 +858,6 @@ complete_start_timer=$starttime
# Purge some log files # Purge some log files
> "$complete_log" > "$complete_log"
> "$test_result"
> "$script_dir/lxc_boot.log" > "$script_dir/lxc_boot.log"
# Initialize LXC network # Initialize LXC network
@ -912,7 +888,6 @@ initialize_values() {
RESULT_check_port=0 RESULT_check_port=0
RESULT_change_url=0 RESULT_change_url=0
RESULT_action_config_panel=0 RESULT_action_config_panel=0
RESULT_template=0
# auto_remove parameter # auto_remove parameter
if [ $interrupt -eq 1 ]; then if [ $interrupt -eq 1 ]; then
@ -939,7 +914,7 @@ initialize_values() {
# Parse the check_process only if it's exist # Parse the check_process only if it's exist
if [ $check_file -eq 1 ] if [ $check_file -eq 1 ]
then then
ECHO_FORMAT "Parsing of check_process file\n" info "Parsing check_process file"
# Remove all commented lines in the check_process # Remove all commented lines in the check_process
sed --in-place '/^#/d' "$check_process" sed --in-place '/^#/d' "$check_process"
@ -996,15 +971,15 @@ then
# Get the value associated to each level # Get the value associated to each level
# Get only the value for the level 5 from the check_process # Get only the value for the level 5 from the check_process
# for i in `seq 1 10` # for i in `seq 1 10`
# do # do
# Find the line for this level # Find the line for this level
# line=$(find_string "^Level $i=") # line=$(find_string "^Level $i=")
line=$(find_string "^Level 5=") line=$(find_string "^Level 5=")
# And get the value # And get the value
# level[$i]=$(echo "$line" | cut -d'=' -f2) # level[$i]=$(echo "$line" | cut -d'=' -f2)
level[5]=$(echo "$line" | cut -d'=' -f2) level[5]=$(echo "$line" | cut -d'=' -f2)
# done # done
# Extract the Options section # Extract the Options section
@ -1082,7 +1057,7 @@ then
line="$(echo "$line" | grep -o "path=.* " | cut -d "=" -f2)" line="$(echo "$line" | grep -o "path=.* " | cut -d "=" -f2)"
# And remove " et spaces to keep only the path. # And remove " et spaces to keep only the path.
line="${line//[\" ]/}" line="${line//[\" ]/}"
# If this path is not empty or equal to /. It's become the new default path value. # If this path is not empty or equal to /. It become the new default path value.
if [ ${#line} -gt 1 ]; then if [ ${#line} -gt 1 ]; then
test_path="$line" test_path="$line"
fi fi
@ -1099,10 +1074,10 @@ then
if echo "$LIGNE" | grep -q "(PATH)"; then # Path dans le manifest if echo "$LIGNE" | grep -q "(PATH)"; then # Path dans le manifest
MANIFEST_PATH=$(echo "$LIGNE" | cut -d '=' -f1) # Récupère la clé du manifest correspondant au path MANIFEST_PATH=$(echo "$LIGNE" | cut -d '=' -f1) # Récupère la clé du manifest correspondant au path
parse_path=$(echo "$LIGNE" | cut -d '"' -f2) # Lit le path du check_process parse_path=$(echo "$LIGNE" | cut -d '"' -f2) # Lit le path du check_process
if [ -n "$parse_path" ]; then # Si le path n'est pas null, utilise ce path au lieu de la valeur par défaut. if [ -n "$parse_path" ]; then # Si le path nest pas null, utilise ce path au lieu de la valeur par défaut.
PATH_TEST=$(echo "$LIGNE" | cut -d '"' -f2) PATH_TEST=$(echo "$LIGNE" | cut -d '"' -f2)
fi fi
LIGNE=$(echo "$LIGNE" | cut -d '(' -f1) # Retire l'indicateur de clé de manifest à la fin de la ligne LIGNE=$(echo "$LIGNE" | cut -d '(' -f1) # Retire lindicateur de clé de manifest à la fin de la ligne
fi fi
# Parse all infos about arguments of actions.toml # Parse all infos about arguments of actions.toml
@ -1246,7 +1221,7 @@ then
done 3<<< "$(grep "^;; " "$check_process")" done 3<<< "$(grep "^;; " "$check_process")"
# No check_process file. Try to parse the manifest. # No check_process file. Try to parse the manifest.
else else
# Initialize the values for this serie of tests # Initialize the values for this serie of tests
initialize_values initialize_values
@ -1320,7 +1295,7 @@ else
# Disable some tests if the manifest key doesn't be found # Disable some tests if the manifest key doesn't be found
if [ -z "$domain_arg" ] if [ -z "$domain_arg" ]
then then
ECHO_FORMAT "The manifest key for domain didn't be find.\n" "lyellow" error "The manifest key for domain was not found."
setup_sub_dir=0 setup_sub_dir=0
count_test "$setup_root" || setup_root=0 count_test "$setup_root" || setup_root=0
count_test "$multi_instance" || multi_instance=0 count_test "$multi_instance" || multi_instance=0
@ -1329,14 +1304,14 @@ else
fi fi
if [ -z "$path_arg" ] if [ -z "$path_arg" ]
then then
ECHO_FORMAT "The manifest key for path didn't be find.\n" "lyellow" error "The manifest key for path was not found."
count_test "$setup_root" || setup_root=0 count_test "$setup_root" || setup_root=0
count_test "$multi_instance" || multi_instance=0 count_test "$multi_instance" || multi_instance=0
count_test "$incorrect_path" || incorrect_path=0 count_test "$incorrect_path" || incorrect_path=0
fi fi
if [ -z "$public_arg" ] if [ -z "$public_arg" ]
then then
ECHO_FORMAT "The manifest key for public didn't be find.\n" "lyellow" error "The manifest key for public was not found."
setup_private=0 setup_private=0
setup_public=0 setup_public=0
all_test=$((all_test-2)) all_test=$((all_test-2))
@ -1525,10 +1500,10 @@ if [ $type_exec_env -ge 1 ] && [ $send_mail -eq 1 ]
then then
# Add a 'from' header for the official CI only. # Add a 'from' header for the official CI only.
# Apparently, this trick is not needed anymore !? # Apparently, this trick is not needed anymore !?
# if [ $type_exec_env -eq 2 ]; then # if [ $type_exec_env -eq 2 ]; then
# from_yuno="-a \"From: yunohost@yunohost.org\"" # from_yuno="-a \"From: yunohost@yunohost.org\""
# fi # fi
# Get the maintainer email from the manifest. If it doesn't found if the check_process # Get the maintainer email from the manifest. If it doesn't found if the check_process
if [ -z "$dest" ]; then if [ -z "$dest" ]; then

72
sub_scripts/common.sh Executable file
View file

@ -0,0 +1,72 @@
#!/bin/bash
readonly NORMAL=$(printf '\033[0m')
readonly BOLD=$(printf '\033[1m')
readonly faint=$(printf '\033[2m')
readonly UNDERLINE=$(printf '\033[4m')
readonly NEGATIVE=$(printf '\033[7m')
readonly RED=$(printf '\033[31m')
readonly GREEN=$(printf '\033[32m')
readonly ORANGE=$(printf '\033[33m')
readonly BLUE=$(printf '\033[34m')
readonly YELLOW=$(printf '\033[93m')
readonly WHITE=$(printf '\033[39m')
function title()
{
cat << EOF | tee -a "$complete_log"
${BOLD}
===================================
$1
===================================
${NORMAL}
EOF
}
function small_title()
{
echo -e "\n${BOLD} > ${1}${NORMAL}\n" | tee -a "$complete_log"
}
function debug()
{
echo "$1" >> "$complete_log"
}
function info()
{
echo "${1}" | tee -a "$complete_log"
}
function success()
{
echo "${BOLD}${GREEN}Success: ${1}${NORMAL}" | tee -a "$complete_log"
}
function warning()
{
echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}" | tee -a "$complete_log" 2>&1
}
function error()
{
echo "${BOLD}${RED}Error: ${1}${NORMAL}" | tee -a "$complete_log" 2>&1
}
function critical()
{
echo "${BOLD}${RED}Critical: ${1}${NORMAL}" | tee -a "$complete_log" 2>&1
}
function report_test_success () {
echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n" | tee -a "$complete_log" 2>&1
}
function report_test_warning () {
echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n" | tee -a "$complete_log" 2>&1
}
function report_test_failed () {
echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n" | tee -a "$complete_log" 2>&1
}

118
sub_scripts/launcher.sh
View file

@ -6,7 +6,8 @@ echo -e "Loads functions from launcher.sh"
# Globals variables # Globals variables
#================================================= #=================================================
arg_ssh="-tt" # -q aims to disable the display of 'Debian GNU/Linux' each time a command is ran
arg_ssh="-tt -q"
snapshot_path="/var/lib/lxcsnaps/$lxc_name" snapshot_path="/var/lib/lxcsnaps/$lxc_name"
current_snapshot=snap0 current_snapshot=snap0
@ -48,14 +49,14 @@ stop_timer () {
[ $minutes -eq 1 ] && pminutes="${pminutes}, " || test -z "$pminutes" || pminutes="${pminutes}s, " [ $minutes -eq 1 ] && pminutes="${pminutes}, " || test -z "$pminutes" || pminutes="${pminutes}s, "
[ $seconds -gt 1 ] && pseconds="${pseconds}s" [ $seconds -gt 1 ] && pseconds="${pseconds}s"
time="${phours}${pminutes}${pseconds} ($(date '+%T'))"
if [ $1 -eq 2 ]; then if [ $1 -eq 2 ]; then
ECHO_FORMAT "Working time for this test: " "blue" info "Working time for this test: $time"
elif [ $1 -eq 3 ]; then elif [ $1 -eq 3 ]; then
ECHO_FORMAT "Global working time for all tests: " "blue" info "Global working time for all tests: $time"
else else
ECHO_FORMAT "Working time: " "blue" info "Working time: $time" >/dev/null
fi fi
ECHO_FORMAT "${phours}${pminutes}${pseconds} ($(date '+%T')).\n" "blue"
} }
#================================================= #=================================================
@ -88,7 +89,7 @@ create_temp_backup () {
then then
echo "snap$snap_number doesn't exist, its first creation can takes a little while." >&2 echo "snap$snap_number doesn't exist, its first creation can takes a little while." >&2
# Create the snapshot. # Create the snapshot.
sudo lxc-snapshot --name $lxc_name >> "$test_result" 2>&1 sudo lxc-snapshot --name $lxc_name >> "$complete_log" 2>&1
# lxc always creates the first snapshot it can creates. # lxc always creates the first snapshot it can creates.
# So if snap1 doesn't exist and you try to create snap2, it will be named snap1. # So if snap1 doesn't exist and you try to create snap2, it will be named snap1.
@ -100,12 +101,12 @@ create_temp_backup () {
fi fi
# Update the snapshot with rsync to clone the current lxc state # Update the snapshot with rsync to clone the current lxc state
sudo rsync --acls --archive --delete --executability --itemize-changes --xattrs "/var/lib/lxc/$lxc_name/rootfs/" "$snapshot_path/snap$snap_number/rootfs/" > /dev/null 2>> "$test_result" sudo rsync --acls --archive --delete --executability --itemize-changes --xattrs "/var/lib/lxc/$lxc_name/rootfs/" "$snapshot_path/snap$snap_number/rootfs/" > /dev/null 2>> "$complete_log"
# Set this snapshot as the current snapshot # Set this snapshot as the current snapshot
current_snapshot=snap$snap_number current_snapshot=snap$snap_number
stop_timer 1 >> $complete_log stop_timer 1
# Restart the container, after the snapshot # Restart the container, after the snapshot
LXC_START "true" >&2 LXC_START "true" >&2
@ -121,9 +122,9 @@ use_temp_snapshot () {
echo "127.0.0.1 $lxc_name" | sudo tee --append "$snapshot_path/$current_snapshot/rootfs/etc/hosts" > /dev/null echo "127.0.0.1 $lxc_name" | sudo tee --append "$snapshot_path/$current_snapshot/rootfs/etc/hosts" > /dev/null
# Restore this snapshot. # Restore this snapshot.
sudo rsync --acls --archive --delete --executability --itemize-changes --xattrs "$snapshot_path/$current_snapshot/rootfs/" "/var/lib/lxc/$lxc_name/rootfs/" > /dev/null 2>> "$test_result" sudo rsync --acls --archive --delete --executability --itemize-changes --xattrs "$snapshot_path/$current_snapshot/rootfs/" "/var/lib/lxc/$lxc_name/rootfs/" > /dev/null 2>> "$complete_log"
stop_timer 1 >> "$complete_log" stop_timer 1
# Retrieve the app id in the log. To manage the app after # Retrieve the app id in the log. To manage the app after
ynh_app_id=$(sudo tac "$yunohost_log" | grep --only-matching --max-count=1 "YNH_APP_INSTANCE_NAME=[^ ]*" | cut --delimiter='=' --fields=2) ynh_app_id=$(sudo tac "$yunohost_log" | grep --only-matching --max-count=1 "YNH_APP_INSTANCE_NAME=[^ ]*" | cut --delimiter='=' --fields=2)
@ -153,13 +154,13 @@ LXC_INIT () {
# Activate the bridge # Activate the bridge
echo "Initialize network for LXC." echo "Initialize network for LXC."
sudo ifup $lxc_bridge --interfaces=/etc/network/interfaces.d/$lxc_bridge | tee --append "$test_result" 2>&1 sudo ifup $lxc_bridge --interfaces=/etc/network/interfaces.d/$lxc_bridge | tee --append "$complete_log" 2>&1
# Activate iptables rules # Activate iptables rules
echo "Activate iptables rules." echo "Activate iptables rules."
sudo iptables --append FORWARD --in-interface $lxc_bridge --out-interface $main_iface --jump ACCEPT | tee --append "$test_result" 2>&1 sudo iptables --append FORWARD --in-interface $lxc_bridge --out-interface $main_iface --jump ACCEPT | tee --append "$complete_log" 2>&1
sudo iptables --append FORWARD --in-interface $main_iface --out-interface $lxc_bridge --jump ACCEPT | tee --append "$test_result" 2>&1 sudo iptables --append FORWARD --in-interface $main_iface --out-interface $lxc_bridge --jump ACCEPT | tee --append "$complete_log" 2>&1
sudo iptables --table nat --append POSTROUTING --source $ip_range.0/24 --jump MASQUERADE | tee --append "$test_result" 2>&1 sudo iptables --table nat --append POSTROUTING --source $ip_range.0/24 --jump MASQUERADE | tee --append "$complete_log" 2>&1
} }
LXC_START () { LXC_START () {
@ -176,11 +177,11 @@ LXC_START () {
# Start the container and log the booting process in $script_dir/lxc_boot.log # Start the container and log the booting process in $script_dir/lxc_boot.log
# Try to start only if the container is not already started # Try to start only if the container is not already started
if ! is_lxc_running; then if ! is_lxc_running; then
echo -n "Start the LXC container" >> "$test_result" debug "Start the LXC container" >> "$complete_log"
sudo lxc-start --name=$lxc_name --daemon --logfile "$script_dir/lxc_boot.log" | tee --append "$test_result" 2>&1 sudo lxc-start --name=$lxc_name --daemon --logfile "$script_dir/lxc_boot.log" | tee --append "$complete_log" 2>&1
local avoid_witness=0 local avoid_witness=0
else else
echo -n "A LXC container is already running" >> "$test_result" debug "A LXC container is already running"
local avoid_witness=1 local avoid_witness=1
fi fi
@ -188,7 +189,7 @@ LXC_START () {
local j=0 local j=0
for j in `seq 1 5` for j in `seq 1 5`
do do
echo -n . >> "$test_result" debug "." >> "$complete_log"
# Try to connect with ssh to check if the container is ready to work. # Try to connect with ssh to check if the container is ready to work.
if ssh $arg_ssh -o ConnectTimeout=10 $lxc_name "exit 0" > /dev/null 2>&1; then if ssh $arg_ssh -o ConnectTimeout=10 $lxc_name "exit 0" > /dev/null 2>&1; then
# Break the for loop if the container is ready. # Break the for loop if the container is ready.
@ -205,23 +206,23 @@ LXC_START () {
local failstart=0 local failstart=0
# Check if the container is running # Check if the container is running
if ! is_lxc_running; then if ! is_lxc_running; then
ECHO_FORMAT "The LXC container didn't start...\n" "red" "bold" critical "The LXC container didn't start..."
failstart=1 failstart=1
if [ $i -ne $max_try ]; then if [ $i -ne $max_try ]; then
ECHO_FORMAT "Rebooting the container...\n" "red" "bold" info "Rebooting the container..."
fi fi
LXC_STOP # Stop the LXC container LXC_STOP
elif ! ssh $arg_ssh -o ConnectTimeout=60 $lxc_name "sudo ping -q -c 2 security.debian.org > /dev/null 2>&1; exit \$?" >> "$test_result" 2>&1
then
# Try to ping security.debian.org to check the connectivity from the container # Try to ping security.debian.org to check the connectivity from the container
ECHO_FORMAT "The container failed to connect to internet...\n" "red" "bold" elif ! ssh $arg_ssh -o ConnectTimeout=60 $lxc_name "sudo ping -q -c 2 security.debian.org > /dev/null 2>&1; exit \$?" >> "$complete_log" 2>&1
then
critical "The container failed to connect to internet..."
failstart=1 failstart=1
if [ $i -ne $max_try ]; then if [ $i -ne $max_try ]; then
ECHO_FORMAT "Rebooting the container...\n" "red" "bold" info "Rebooting the container..."
fi fi
LXC_STOP # Stop the LXC container LXC_STOP
else
# Create files to check if the remove script does not remove them accidentally # Create files to check if the remove script does not remove them accidentally
else
[ $avoid_witness -eq 0 ] && set_witness_files [ $avoid_witness -eq 0 ] && set_witness_files
# Break the for loop if the container is ready. # Break the for loop if the container is ready.
@ -238,16 +239,16 @@ LXC_START () {
ci_path=$(grep "CI_URL=" "$script_dir/../config" | cut -d= -f2) ci_path=$(grep "CI_URL=" "$script_dir/../config" | cut -d= -f2)
local subject="[YunoHost] Container in trouble on $ci_path." local subject="[YunoHost] Container in trouble on $ci_path."
local message="The container failed to start $max_try times on $ci_path. local message="The container failed to start $max_try times on $ci_path.
$lxc_check_result $lxc_check_result
Please have a look to the log of lxc_check: Please have a look to the log of lxc_check:
$(cat "$script_dir/lxc_check.log")" $(cat "$script_dir/lxc_check.log")"
if [ $lxc_check -eq 2 ]; then if [ $lxc_check -eq 2 ]; then
# Add the log of lxc_build # Add the log of lxc_build
message="$message message="$message
Here the log of lxc_build: Here the log of lxc_build:
$(cat "$script_dir/sub_scripts/Build_lxc.log")" $(cat "$script_dir/sub_scripts/Build_lxc.log")"
fi fi
dest=$(grep 'dest=' "$script_dir/../config" | cut -d= -f2) dest=$(grep 'dest=' "$script_dir/../config" | cut -d= -f2)
@ -255,58 +256,53 @@ $(cat "$script_dir/sub_scripts/Build_lxc.log")"
fi fi
} }
ECHO_FORMAT "The container failed to start $max_try times...\n" "red" "bold" critical "The container failed to start $max_try times..."
ECHO_FORMAT "Boot log:\n" clog info "Boot log:\n"
cat "$script_dir/lxc_boot.log" | tee --append "$test_result" cat "$script_dir/lxc_boot.log" | tee --append "$complete_log"
ECHO_FORMAT "lxc_check will try to fix the container...\n" "red" "bold" info "lxc_check will try to fix the container..."
$script_dir/sub_scripts/lxc_check.sh --no-lock | tee "$script_dir/lxc_check.log" $script_dir/sub_scripts/lxc_check.sh --no-lock | tee "$script_dir/lxc_check.log"
# PIPESTATUS is an array with the exit code of each command followed by a pipe # PIPESTATUS is an array with the exit code of each command followed by a pipe
local lxc_check=${PIPESTATUS[0]} local lxc_check=${PIPESTATUS[0]}
LXC_INIT LXC_INIT
if [ $lxc_check -eq 0 ]; then if [ $lxc_check -eq 0 ]; then
local lxc_check_result="The container seems to be ok, according to lxc_check." local lxc_check_result="The container seems to be ok, according to lxc_check."
ECHO_FORMAT "$lxc_check_result\n" "lgreen" "bold" success "$lxc_check_result"
send_email send_email
i=0 i=0
elif [ $lxc_check -eq 1 ]; then elif [ $lxc_check -eq 1 ]; then
local lxc_check_result="An error has happened with the host. Please check the configuration." local lxc_check_result="An error has happened with the host. Please check the configuration."
ECHO_FORMAT "$lxc_check_result\n" "red" "bold" critical "$lxc_check_result"
send_email send_email
stop_timer 1 >> "$complete_log" stop_timer 1
return 1 return 1
elif [ $lxc_check -eq 2 ]; then elif [ $lxc_check -eq 2 ]; then
local lxc_check_result="The container is broken, it will be rebuilt." local lxc_check_result="The container is broken, it will be rebuilt."
ECHO_FORMAT "$lxc_check_result\n" "red" "bold" critical "$lxc_check_result"
$script_dir/sub_scripts/lxc_build.sh $script_dir/sub_scripts/lxc_build.sh
LXC_INIT LXC_INIT
send_email send_email
i=0 i=0
elif [ $lxc_check -eq 3 ]; then elif [ $lxc_check -eq 3 ]; then
local lxc_check_result="The container has been fixed by lxc_check." local lxc_check_result="The container has been fixed by lxc_check."
ECHO_FORMAT "$lxc_check_result\n" "lgreen" "bold" success "$lxc_check_result"
send_email send_email
i=0 i=0
fi fi
fi fi
done done
stop_timer 1 >> "$complete_log" stop_timer 1
start_timer start_timer
# Count the number of lines of the current yunohost log file.
COPY_LOG 1
# Copy the package into the container. # Copy the package into the container.
rsync -rq --delete "$package_path" "$lxc_name": >> "$test_result" 2>&1 rsync -rq --delete "$package_path" "$lxc_name": >> "$complete_log" 2>&1
# Execute the command given in argument in the container and log its results. # Execute the command given in argument in the container and log its results.
ssh $arg_ssh $lxc_name "$1 > /dev/null 2>> temp_yunohost-cli.log; exit \$?" >> "$test_result" 2>&1 ssh $arg_ssh $lxc_name "$1; exit $?" | tee -a "$complete_log"
# Store the return code of the command # Store the return code of the command
local returncode=$? local returncode=${PIPESTATUS[0]}
# Retrieve the log of the previous command and copy its content in the temporary log stop_timer 1
sudo cat "/var/lib/lxc/$lxc_name/rootfs/home/pchecker/temp_yunohost-cli.log" >> "$temp_log"
stop_timer 1 >> "$complete_log"
# Return the exit code of the ssh command # Return the exit code of the ssh command
return $returncode return $returncode
} }
@ -317,8 +313,8 @@ LXC_STOP () {
start_timer start_timer
# Stop the LXC container # Stop the LXC container
if is_lxc_running; then if is_lxc_running; then
echo "Stop the LXC container" >> "$test_result" debug "Stop the LXC container"
sudo lxc-stop --name=$lxc_name | tee --append "$test_result" 2>&1 sudo lxc-stop --name=$lxc_name | tee --append "$complete_log" 2>&1
fi fi
# Fix the missing hostname in the hosts file # Fix the missing hostname in the hosts file
@ -337,9 +333,9 @@ LXC_STOP () {
fi fi
# Restore the snapshot. # Restore the snapshot.
echo "Restore the previous snapshot." >> "$test_result" debug "Restore the previous snapshot."
sudo rsync --acls --archive --delete --executability --itemize-changes --xattrs "$snapshot_path/$current_snapshot/rootfs/" "/var/lib/lxc/$lxc_name/rootfs/" > /dev/null 2>> "$test_result" sudo rsync --acls --archive --delete --executability --itemize-changes --xattrs "$snapshot_path/$current_snapshot/rootfs/" "/var/lib/lxc/$lxc_name/rootfs/" > /dev/null 2>> "$complete_log"
stop_timer 1 >> "$complete_log" stop_timer 1
} }
LXC_TURNOFF () { LXC_TURNOFF () {
@ -348,21 +344,21 @@ LXC_TURNOFF () {
echo "Disable iptables rules." echo "Disable iptables rules."
if sudo iptables --check FORWARD --in-interface $lxc_bridge --out-interface $main_iface --jump ACCEPT 2> /dev/null if sudo iptables --check FORWARD --in-interface $lxc_bridge --out-interface $main_iface --jump ACCEPT 2> /dev/null
then then
sudo iptables --delete FORWARD --in-interface $lxc_bridge --out-interface $main_iface --jump ACCEPT >> "$test_result" 2>&1 sudo iptables --delete FORWARD --in-interface $lxc_bridge --out-interface $main_iface --jump ACCEPT >> "$complete_log" 2>&1
fi fi
if sudo iptables --check FORWARD --in-interface $main_iface --out-interface $lxc_bridge --jump ACCEPT 2> /dev/null if sudo iptables --check FORWARD --in-interface $main_iface --out-interface $lxc_bridge --jump ACCEPT 2> /dev/null
then then
sudo iptables --delete FORWARD --in-interface $main_iface --out-interface $lxc_bridge --jump ACCEPT | tee --append "$test_result" 2>&1 sudo iptables --delete FORWARD --in-interface $main_iface --out-interface $lxc_bridge --jump ACCEPT | tee --append "$complete_log" 2>&1
fi fi
if sudo iptables --table nat --check POSTROUTING --source $ip_range.0/24 --jump MASQUERADE 2> /dev/null if sudo iptables --table nat --check POSTROUTING --source $ip_range.0/24 --jump MASQUERADE 2> /dev/null
then then
sudo iptables --table nat --delete POSTROUTING --source $ip_range.0/24 --jump MASQUERADE | tee --append "$test_result" 2>&1 sudo iptables --table nat --delete POSTROUTING --source $ip_range.0/24 --jump MASQUERADE | tee --append "$complete_log" 2>&1
fi fi
echo "Disable the network bridge." echo "Disable the network bridge."
if sudo ifquery $lxc_bridge --state > /dev/null if sudo ifquery $lxc_bridge --state > /dev/null
then then
sudo ifdown --force $lxc_bridge | tee --append "$test_result" 2>&1 sudo ifdown --force $lxc_bridge | tee --append "$complete_log" 2>&1
fi fi
# Set snap0 as the current snapshot # Set snap0 as the current snapshot

166
sub_scripts/log_extractor.sh
View file

@ -1,166 +0,0 @@
#!/bin/bash
echo "Loads functions from log_extractor.sh"
ECHO_FORMAT () {
# Simply an echo with color and typo
# $2 = color
# $3 = typo
# Last arg = clog
if [ "$2" == "red" ]; then
echo -en "\e[91m"
fi
if [ "$2" == "lyellow" ]; then
echo -en "\e[93m"
fi
if [ "$2" == "lgreen" ]; then
echo -en "\e[92m"
fi
if [ "$2" == "blue" ]; then
echo -en "\e[94m"
fi
if [ "$2" == "white" ]; then
echo -en "\e[97m"
fi
if [ "$3" == "bold" ]; then
echo -en "\e[1m"
fi
if [ "$3" == "underlined" ]; then
echo -en "\e[4m"
fi
local copy_log=--
# If 'clog' is given as argument, the echo command will be duplicated into the complete log.
if [ "$2" == "clog" ] || [ "$3" == "clog" ] || [ "$4" == "clog" ]; then
copy_log="$complete_log"
fi
echo -en "$1" | tee -a "$test_result" "$copy_log"
echo -en "\e[0m"
}
COPY_LOG () {
# Extract A small part of $yunohost_log.
# $1 = 1 or 2. If '1', count the number of line of the current log file.
# If '2', copy the log from the last read line.
if [ $1 -eq 1 ]; then
# Count the number of lines in YunoHost log
log_line=$(sudo wc --lines "$yunohost_log" | cut --delimiter=' ' --fields=1)
# Ignore the first line, it's duplicated of the previous log
log_line=$(( $log_line + 1 ))
# Erase the temporary log
> "$temp_log"
fi
if [ $1 -eq 2 ]; then
# Copy the log from the last read line
sudo tail --lines=+$log_line "$yunohost_log" >> "$temp_log"
fi
}
FALSE_ERRORS_DETECTION () {
# Detect false positive errors.
false_positive_error=0
# Detect network error
if grep --quiet "Network is unreachable" "$temp_result"
then
false_positive_error=1
false_positive_error_cond="network"
fi
# Detect DNS error
if grep --quiet "Temporary failure resolving" "$temp_result"
then
false_positive_error=1
false_positive_error_cond="DNS failure"
fi
if grep --quiet "unable to resolve host address" "$temp_result"
then
false_positive_error=1
false_positive_error_cond="DNS failure"
fi
# Detect Corrupt source
if grep --quiet "Corrupt source" "$temp_result"
then
false_positive_error=1
false_positive_error_cond="corrupt source"
fi
}
PARSE_LOG () {
# Print all errors and warning found in the log.
while read log_read_line
do
if echo "$log_read_line" | grep --quiet "^>ERROR: "; then
# Print a red "Error"
ECHO_FORMAT "Error:" "red" "underlined"
# And print the error itself
ECHO_FORMAT " $(echo "$log_read_line\n" | sed 's/^>ERROR: //')" "red"
YUNOHOST_RESULT=1
YUNOHOST_REMOVE=1
fi
if echo "$log_read_line" | grep --quiet "^>WARNING: "; then
# Print a yellow "Warning:"
ECHO_FORMAT "Warning:" "lyellow" "underlined"
# And print the warning itself
ECHO_FORMAT " $(echo "$log_read_line\n" | sed 's/^>WARNING: //')" "lyellow"
fi
if echo "$log_read_line" | grep --quiet "^>INFO: "; then
# Print a white "Info:"
ECHO_FORMAT "Info:" "white" "underlined"
# And print the info itself
ECHO_FORMAT " $(echo "$log_read_line\n" | sed 's/^>INFO: //')" "white"
fi
done < "$temp_result"
}
CLEAR_LOG () {
# Remove all knew useless warning lines.
# Useless warnings from wget
sed --in-place '/^>WARNING: yunohost\.hook <lambda> - \[[[:digit:].]*\] *$/d' "$temp_result" # Empty line foregoing wget progression
sed --in-place '/^>WARNING: yunohost\.hook <lambda> - \[[[:digit:].]*\] *[[:digit:]]*K \.* /d' "$temp_result" # Wget progression
sed --in-place '/% Total % Received % Xferd/d' "$temp_result" # Wget statistics
sed --in-place '/Dload Upload Total Spent/d' "$temp_result" # Wget statistics (again)
sed --in-place '/--:--:-- --:--:-- --:--:--/d' "$temp_result" # Wget statistics (Yes, again...)
# Useless warning from yunohost backup.
sed --in-place '/^>WARNING: yunohost.backup backup_restore - \[[[:digit:].]*\] YunoHost est déjà installé$/d' "$temp_result"
# "processing action" INFO lines
sed --in-place '/^>INFO: .* - processing action/d' "$temp_result"
# Clean INFO lines
sed --in-place 's/^\(>INFO: \)yunohost\.hook <lambda> - \[[[:digit:].]*\]/\1/g' "$temp_result"
# "This feature is experimental"
sed --in-place '/^>WARNING: .* This feature is experimental and not considered stable/d' "$temp_result"
# "moulinette.core translate"
sed --in-place '/^>ERROR: moulinette.core translate/d' "$temp_result"
# Empty lines
sed --in-place '/^$/d' "$temp_result"
}
LOG_EXTRACTOR () {
# Analyse the log to extract "warning" and "error" lines
# Copy the log from the last read line.
COPY_LOG 2
# Erase the temporary result file
> "$temp_result"
# Duplicate the part of the yunohost log into the complete log.
cat "$temp_log" >> "$complete_log"
# Find all errors, warnings and infos in the log file
grep --extended-regexp " ERROR | WARNING | INFO |yunohost.*: error:" "$temp_log" >> "$temp_result"
sed -i 's/^.* ERROR */>ERROR: /' "$temp_result"
sed -i 's/^.* WARNING */>WARNING: /' "$temp_result"
sed -i 's/^.* INFO */>INFO: /' "$temp_result"
CLEAR_LOG # Remove all knew useless warning lines.
PARSE_LOG # Print all errors and warning found in the log.
FALSE_ERRORS_DETECTION # Detect if there's a temporary error that shouldn't impact the test.
}

2
sub_scripts/lxc_build.sh
View file

@ -83,7 +83,7 @@ sudo apt-get update >> "$LOG_BUILD_LXC" 2>&1
sudo apt-get install -y lxc lxctl >> "$LOG_BUILD_LXC" 2>&1 sudo apt-get install -y lxc lxctl >> "$LOG_BUILD_LXC" 2>&1
echo -e "\e[1m> Install git, curl and lynx\e[0m" | tee -a "$LOG_BUILD_LXC" echo -e "\e[1m> Install git, curl and lynx\e[0m" | tee -a "$LOG_BUILD_LXC"
sudo apt-get install -y git curl lynx jq python3-pip >> "$LOG_BUILD_LXC" 2>&1 sudo apt-get install -y git curl lynx jq python3-pip boxes >> "$LOG_BUILD_LXC" 2>&1
# For those who have disabled APT::Install-Recommends we need to manually install the following packages. # For those who have disabled APT::Install-Recommends we need to manually install the following packages.
sudo apt-get install -y debootstrap rsync bridge-utils >> "$LOG_BUILD_LXC" 2>&1 sudo apt-get install -y debootstrap rsync bridge-utils >> "$LOG_BUILD_LXC" 2>&1

527
sub_scripts/testing_process.sh
View file

File diff suppressed because it is too large Load diff