From 345b60fe324a96827964f01ba28643d4e64adada Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 17 Dec 2020 18:43:33 +0100 Subject: [PATCH] Moar fiksize --- package_check.sh | 131 +++++----- sub_scripts/build_base_lxc.sh | 53 ++++ sub_scripts/common.sh | 31 ++- sub_scripts/lxc.sh | 4 - sub_scripts/lxc_build.sh | 72 ----- sub_scripts/lxc_remove.sh | 41 --- sub_scripts/testing_process.sh | 464 ++++++++++++++------------------- sub_scripts/witness.sh | 92 +++++++ 8 files changed, 428 insertions(+), 460 deletions(-) create mode 100755 sub_scripts/build_base_lxc.sh delete mode 100755 sub_scripts/lxc_build.sh delete mode 100755 sub_scripts/lxc_remove.sh create mode 100644 sub_scripts/witness.sh diff --git a/package_check.sh b/package_check.sh index e303a49..9d0ef7d 100755 --- a/package_check.sh +++ b/package_check.sh @@ -40,14 +40,9 @@ exit 0 clean_exit () { - # Exit and remove all temp files - # $1 = exit code LXC_RESET - # Remove temporary files rm -rf "$TEST_CONTEXT" - - # Remove the lock file rm -f "$lock_file" exit $1 @@ -64,71 +59,77 @@ clean_exit () { gitbranch="" force_install_ok=0 interactive=0 -arguments=("$@") -getopts_built_arg=() -# Read the array value per value -for i in `seq 0 $(( ${#arguments[@]} -1 ))` -do - if [[ "${arguments[$i]}" =~ "--branch=" ]] - then - getopts_built_arg+=(-b) - arguments[$i]=${arguments[$i]//--branch=/} - fi - # For each argument in the array, reduce to short argument for getopts - arguments[$i]=${arguments[$i]//--interactive/-i} - arguments[$i]=${arguments[$i]//--help/-h} - getopts_built_arg+=("${arguments[$i]}") -done +function parse_args() { -# Read and parse all the arguments -# Use a function here, to use standart arguments $@ and be able to use shift. -parse_arg () { - while [ $# -ne 0 ] + local getopts_built_arg=() + + # Read the array value per value + for i in `seq 0 $(( ${#arguments[@]} -1 ))` do - # If the paramater begins by -, treat it with getopts - if [ "${1:0:1}" == "-" ] + if [[ "${arguments[$i]}" =~ "--branch=" ]] then - # Initialize the index of getopts - OPTIND=1 - # Parse with getopts only if the argument begin by - - getopts ":b:fihlyr" parameter || true - case $parameter in - b) - # --branch=branch-name - gitbranch="-b $OPTARG" - shift_value=2 - ;; - i) - # --interactive - interactive=1 - shift_value=1 - ;; - h) - # --help - print_help - ;; - \?) - echo "Invalid argument: -${OPTARG:-}" - print_help - ;; - :) - echo "-$OPTARG parameter requires an argument." - print_help - ;; - esac - # Otherwise, it's not an option, it's an operand - else - path_to_package_to_test="$1" - shift_value=1 + getopts_built_arg+=(-b) + arguments[$i]=${arguments[$i]//--branch=/} fi - # Shift the parameter and its argument - shift $shift_value + # For each argument in the array, reduce to short argument for getopts + arguments[$i]=${arguments[$i]//--interactive/-i} + arguments[$i]=${arguments[$i]//--help/-h} + getopts_built_arg+=("${arguments[$i]}") done + + # Read and parse all the arguments + # Use a function here, to use standart arguments $@ and be able to use shift. + parse_arg () { + while [ $# -ne 0 ] + do + # If the paramater begins by -, treat it with getopts + if [ "${1:0:1}" == "-" ] + then + # Initialize the index of getopts + OPTIND=1 + # Parse with getopts only if the argument begin by - + getopts ":b:fihlyr" parameter || true + case $parameter in + b) + # --branch=branch-name + gitbranch="-b $OPTARG" + shift_value=2 + ;; + i) + # --interactive + interactive=1 + shift_value=1 + ;; + h) + # --help + print_help + ;; + \?) + echo "Invalid argument: -${OPTARG:-}" + print_help + ;; + :) + echo "-$OPTARG parameter requires an argument." + print_help + ;; + esac + # Otherwise, it's not an option, it's an operand + else + path_to_package_to_test="$1" + shift_value=1 + fi + # Shift the parameter and its argument + shift $shift_value + done + } + + # Call parse_arg and pass the modified list of args as a array of arguments. + parse_arg "${getopts_built_arg[@]}" } -# Call parse_arg and pass the modified list of args as a array of arguments. -parse_arg "${getopts_built_arg[@]}" +arguments=("$@") +parse_args #================================================= # Check if the lock file exist @@ -158,12 +159,14 @@ echo "start:$(date +%s):$$" > "$lock_file" # Various logistic checks and upgrades... #================================================= -assert_we_are_the_setup_user assert_we_are_connected_to_the_internets -#self_upgrade + +#self_upgrade # FIXME renenable this later fetch_or_upgrade_package_linter # Reset and create a fresh container to work with +check_lxd_setup +lxc image list $LXC_BASE | grep -q -w $LXC_BASE || log_critical "The base image $LXC_BASE doesn't exist yet. Consider using the build_base_lxc.sh to create it first" LXC_RESET LXC_CREATE diff --git a/sub_scripts/build_base_lxc.sh b/sub_scripts/build_base_lxc.sh new file mode 100755 index 0000000..9c7ebcd --- /dev/null +++ b/sub_scripts/build_base_lxc.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +cd $(dirname $(realpath $0) | sed 's@/sub_scripts$@@g') +source "./sub_scripts/common.sh" + +function rebuild_base_lxc() +{ + check_lxd_setup + + set -x + sudo lxc info $LXC_BASE >/dev/null && sudo lxc delete $LXC_BASE --force + sudo lxc launch images:debian/$DIST/$ARCH $LXC_BASE + sudo lxc config set $LXC_BASE security.privileged true + sudo lxc config set $LXC_BASE security.nesting true # Need this for apparmor for some reason + sudo lxc restart $LXC_BASE + sleep 5 + + IN_LXC="sudo lxc exec $LXC_BASE" + + INSTALL_SCRIPT="https://install.yunohost.org/$DIST" + $IN_LXC apt install curl -y + $IN_LXC /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a $YNH_BRANCH" + + $IN_LXC systemctl -q stop apt-daily.timer + $IN_LXC systemctl -q stop apt-daily-upgrade.timer + $IN_LXC systemctl -q stop apt-daily.service + $IN_LXC systemctl -q stop apt-daily-upgrade.service + $IN_LXC systemctl -q disable apt-daily.timer + $IN_LXC systemctl -q disable apt-daily-upgrade.timer + $IN_LXC systemctl -q disable apt-daily.service + $IN_LXC systemctl -q disable apt-daily-upgrade.service + $IN_LXC rm -f /etc/cron.daily/apt-compat + $IN_LXC cp /bin/true /usr/lib/apt/apt.systemd.daily + + # Disable password strength check + $IN_LXC yunohost tools postinstall --domain $DOMAIN --password $YUNO_PWD --force-password + + $IN_LXC yunohost settings set security.password.admin.strength -v -1 + $IN_LXC yunohost settings set security.password.user.strength -v -1 + + $IN_LXC yunohost domain add $SUBDOMAIN + TEST_USER_DISPLAY=${TEST_USER//"_"/""} + $IN_LXC yunohost user create $TEST_USER --firstname $TEST_USER_DISPLAY --mail $TEST_USER@$DOMAIN --lastname $TEST_USER_DISPLAY --password '$YUNO_PWD' + + $IN_LXC yunohost --version + + sudo lxc stop $LXC_BASE + sudo lxc image delete $LXC_BASE + sudo lxc publish $LXC_BASE --alias $LXC_BASE + set +x +} + +rebuild_base_lxc 2>&1 | tee -a "./build_base_lxc.log" diff --git a/sub_scripts/common.sh b/sub_scripts/common.sh index 28c108c..2392ea5 100755 --- a/sub_scripts/common.sh +++ b/sub_scripts/common.sh @@ -1,6 +1,7 @@ #!/bin/bash -DEFAULT_DIST="buster" +ARCH="amd64" +DIST="buster" # By default we'll install Yunohost with the default branch YNH_INSTALL_SCRIPT_BRANCH="" @@ -15,7 +16,8 @@ SUBDOMAIN="sub.$DOMAIN" # User de test TEST_USER="package_checker" -LXC_NAME="ynh-appci-$DEFAULT_DIST" +LXC_BASE="ynh-appci-$DIST-$ARCH-base" +LXC_NAME="ynh-appci-$DIST" [[ -e "./config" ]] && source "./config" @@ -25,20 +27,29 @@ readonly lock_file="./pcheck.lock" # LXC helpers #================================================= -assert_we_are_the_setup_user() { - [ -e "./.setup_user" ] || return - local setup_user=$(cat "./.setup_user") - - [ "$(whoami)" == $setup_user ] \ - || log_critical "Ce script doit être exécuté avec l'utilisateur $setup_user !\nL'utilisateur actuel est $(whoami)." -} - assert_we_are_connected_to_the_internets() { ping -q -c 2 yunohost.org > /dev/null 2>&1 \ || ping -q -c 2 framasoft.org > /dev/null 2>&1 \ || log_critical "Unable to connect to internet." } +function check_lxd_setup() +{ + # Check lxd is installed somehow + [[ -e /snap/bin/lxd ]] || which lxd &>/dev/null \ + || critical "You need to have LXD installed. Refer to the README to know how to install it." + + # Check that we'll be able to use lxc/lxd using sudo (for which the PATH is defined in /etc/sudoers and probably doesn't include /snap/bin) + if [[ ! -e /usr/bin/lxc ]] && [[ ! -e /usr/bin/lxd ]] + then + [[ -e /usr/local/bin/lxc ]] && [[ -e /usr/local/bin/lxd ]] \ + || critical "You might want to add lxc and lxd inside /usr/local/bin so that there's no tricky PATH issue with sudo. If you installed lxd/lxc with snapd, this should do the trick: sudo ln -s /snap/bin/lxc /usr/local/bin/lxc && sudo ln -s /snap/bin/lxd /usr/local/bin/lxd" + fi + + ip a | grep -q lxdbr0 \ + || critical "There is no 'lxdbr0' interface... Did you ran 'lxd init' ?" +} + #================================================= # Logging helpers #================================================= diff --git a/sub_scripts/lxc.sh b/sub_scripts/lxc.sh index 799c7b0..7aa1e0a 100755 --- a/sub_scripts/lxc.sh +++ b/sub_scripts/lxc.sh @@ -60,10 +60,6 @@ LXC_START () { start_timer - # Copy the package into the container. - lxc exec $LXC_NAME -- rm -rf /app_folder - lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet - # Execute the command given in argument in the container and log its results. lxc exec $LXC_NAME --env PACKAGE_CHECK_EXEC=1 -t -- $cmd | tee -a "$complete_log" diff --git a/sub_scripts/lxc_build.sh b/sub_scripts/lxc_build.sh deleted file mode 100755 index 7023c28..0000000 --- a/sub_scripts/lxc_build.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -cd $(dirname $(realpath $0) | sed 's@/sub_scripts$@@g') -source "./sub_scripts/common.sh" - -function check_lxd_setup() -{ - # Check lxd is installed somehow - [[ -e /snap/bin/lxd ]] || which lxd &>/dev/null \ - || critical "You need to have LXD installed. Refer to the README to know how to install it." - - # Check that we'll be able to use lxc/lxd using sudo (for which the PATH is defined in /etc/sudoers and probably doesn't include /snap/bin) - if [[ ! -e /usr/bin/lxc ]] && [[ ! -e /usr/bin/lxd ]] - then - [[ -e /usr/local/bin/lxc ]] && [[ -e /usr/local/bin/lxd ]] \ - || critical "You might want to add lxc and lxd inside /usr/local/bin so that there's no tricky PATH issue with sudo. If you installed lxd/lxc with snapd, this should do the trick: sudo ln -s /snap/bin/lxc /usr/local/bin/lxc && sudo ln -s /snap/bin/lxd /usr/local/bin/lxd" - fi - - ip a | grep -q lxdbr0 \ - || critical "There is no 'lxdbr0' interface... Did you ran 'lxd init' ?" -} - -function rebuild_ynh_appci_base() -{ - check_lxd_setup - - local DIST=${1:-$DEFAULT_DIST} - local BOX=${2:-ynh-appci}-${DIST} - - set -x - sudo lxc info $BOX-base >/dev/null && sudo lxc delete $BOX-base --force - sudo lxc launch images:debian/$DIST/$ARCH $BOX-base - sudo lxc config set $BOX-base security.privileged true - sudo lxc config set $BOX-base security.nesting true # Need this for apparmor for some reason - sudo lxc restart $BOX-base - sleep 5 - - IN_LXC="sudo lxc exec $BOX-base -- /bin/bash -c" - - INSTALL_SCRIPT="https://install.yunohost.org/$DIST" - $IN_LXC "apt install curl -y" - $IN_LXC "curl $INSTALL_SCRIPT | bash -s -- -a $YNH_BRANCH" - - $IN_LXC "systemctl -q stop apt-daily.timer" - $IN_LXC "systemctl -q stop apt-daily-upgrade.timer" - $IN_LXC "systemctl -q stop apt-daily.service" - $IN_LXC "systemctl -q stop apt-daily-upgrade.service " - $IN_LXC "systemctl -q disable apt-daily.timer" - $IN_LXC "systemctl -q disable apt-daily-upgrade.timer" - $IN_LXC "systemctl -q disable apt-daily.service" - $IN_LXC "systemctl -q disable apt-daily-upgrade.service" - $IN_LXC "rm -f /etc/cron.daily/apt-compat" - $IN_LXC "cp /bin/true /usr/lib/apt/apt.systemd.daily" - - # Disable password strength check - $IN_LXC "yunohost tools postinstall --domain $DOMAIN --password $YUNO_PWD --force-password" - - $IN_LXC "yunohost settings set security.password.admin.strength -v -1" - $IN_LXC "yunohost settings set security.password.user.strength -v -1" - - $IN_LXC "yunohost domain add $SUBDOMAIN" - TEST_USER_DISPLAY=${TEST_USER//"_"/""} - $IN_LXC "yunohost user create $TEST_USER --firstname $TEST_USER_DISPLAY --mail $TEST_USER@$DOMAIN --lastname $TEST_USER_DISPLAY --password '$YUNO_PWD'" - - $IN_LXC "yunohost --version" - - sudo lxc stop $BOX-base - sudo lxc publish $BOX-base --alias $BOX-base - set +x -} - -rebuild_ynh_appci_base 2>&1 | tee -a "./lxc_build.log" diff --git a/sub_scripts/lxc_remove.sh b/sub_scripts/lxc_remove.sh deleted file mode 100755 index f29f703..0000000 --- a/sub_scripts/lxc_remove.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -cd $(dirname $(realpath $0) | sed 's@/sub_scripts$@@g') -source "./sub_scripts/common.sh" - -# Check user -assert_we_are_the_setup_user - -touch "$lock_file" - - -log_title "Retire l'ip forwarding." - -sudo rm -f /etc/sysctl.d/lxc_pchecker.conf -sudo sysctl -p - - -log_title "Désactive le bridge réseau" - -sudo ifdown --force $LXC_BRIDGE - - -log_title "Supprime le brige réseau" - -sudo rm -f /etc/network/interfaces.d/$LXC_BRIDGE - - -log_title "Suppression de la machine et de son snapshots" - -sudo lxc-snapshot -n $LXC_NAME -d snap0 -for SNAP in $(sudo ls $LXC_SNAPSHOTS/snap_*install 2>/dev/null) -do - sudo lxc-snapshot -n $LXC_NAME -d $(basename $SNAP) -done -sudo rm -f /var/lib/lxcsnaps/$LXC_NAME/snap0.tar.gz -sudo lxc-destroy -n $LXC_NAME -f - -log_title "Suppression des lignes de pchecker_lxc dans $HOME/.ssh/config" - -BEGIN_LINE=$(cat $HOME/.ssh/config | grep -n "^# ssh pchecker_lxc$" | cut -d':' -f 1 | tail -n1) -sed -i "$BEGIN_LINE,/^IdentityFile/d" $HOME/.ssh/config diff --git a/sub_scripts/testing_process.sh b/sub_scripts/testing_process.sh index c0a2e1e..f17d31d 100755 --- a/sub_scripts/testing_process.sh +++ b/sub_scripts/testing_process.sh @@ -1,6 +1,91 @@ #!/bin/bash +source sub_scripts/witness.sh + #================================================= +# Misc test helpers & coordination +#================================================= + +RUN_ALL_TESTS() { + # Launch all tests successively + curl_error=0 + + # Be sure that the container is running + LXC_START "true" + + # Print the version of YunoHost from the LXC container + log_small_title "YunoHost versions" + LXC_START "sudo yunohost --version" + + # Init the value for the current test + current_test_number=1 + + # The list of test contains for example "TEST_UPGRADE some_commit_id + for testfile in $(ls $TEST_CONTEXT/tests/*.json); + do + TEST_LAUNCHER $testfile + done + +} + +TEST_LAUNCHER () { + local testfile="$1" + + # Start the timer for this test + start_timer + # And keep this value separately + local global_start_timer=$starttime + + current_test_id=$(basename $testfile | cut -d. -f1) + current_test_infos="$TEST_CONTEXT/tests/$current_test_id.json" + current_test_results="$TEST_CONTEXT/results/$current_test_id.json" + echo "{}" > $current_test_results + + local test_type=$(jq -r '.test_type' $testfile) + local test_arg=$(jq -r '.test_arg' $testfile) + + # Execute the test + $test_type $test_arg + + [ $? -eq 0 ] && SET_RESULT "success" main_result || SET_RESULT "failure" main_result + + break_before_continue + + # Restore the started time for the timer + starttime=$global_start_timer + # End the timer for the test + stop_timer 2 + + LXC_STOP + + # Update the lock file with the date of the last finished test. + # $$ is the PID of package_check itself. + echo "$1 $2:$(date +%s):$$" > "$lock_file" +} + +SET_RESULT() { + local result=$1 + local name=$2 + [ "$result" == "success" ] && log_report_test_success || log_report_test_failed + local current_results="$(cat $current_test_results)" + echo "$current_results" | jq --arg result $result ".$name=\$result" > $current_test_results +} + +#================================================= + +at_least_one_install_succeeded () { + + for TEST in $(ls $TEST_CONTEXT/tests/*.json) + do + local test_id=$(basename $TEST | cut -d. -f1) + jq -e '. | select(.test_type == "TEST_INSTALL")' $TEST >/dev/null \ + && jq -e '. | select(.main_result == "success")' $TEST_CONTEXT/results/$test_id.json >/dev/null \ + && return 0 + done + + log_error "All installs failed, therefore the following tests cannot be performed..." + return 1 +} break_before_continue () { @@ -26,6 +111,10 @@ RUN_YUNOHOST_CMD() { log_debug "Running yunohost $1" + # Copy the package into the container. + lxc exec $LXC_NAME -- rm -rf /app_folder + lxc file push -p -r "$package_path" $LXC_NAME/app_folder --quiet + # --output-as none is to disable the json-like output for some commands like backup create LXC_START "yunohost --output-as none --debug $1" \ | grep --line-buffered -v --extended-regexp '^[0-9]+\s+.{1,15}DEBUG' \ @@ -53,6 +142,15 @@ default_install_path() { this_is_a_web_app && echo "/" || echo "" } +path_to_install_type() { + local check_path="$1" + + [ -z "$check_path" ] && echo "nourl" \ + || [ "$check_path" == "/" ] && echo "root" \ + || echo "subdir" + +} + #================================================= # Install and remove an app #================================================= @@ -96,15 +194,6 @@ INSTALL_APP () { return $ret } -path_to_install_type() { - local check_path="$1" - - [ -z "$check_path" ] && echo "nourl" \ - || [ "$check_path" == "/" ] && echo "root" \ - || echo "subdir" - -} - LOAD_SNAPSHOT_OR_INSTALL_APP () { local check_path="$1" @@ -285,15 +374,70 @@ VALIDATE_THAT_APP_CAN_BE_ACCESSED () { && SET_RESULT "failure" alias_traversal [ "$curl_error" -eq 0 ] || return 1 - [ "$install_type" != "private" ] && [ $fell_on_sso_portal -eq 0 ] || return 2 - [ "$install_type" == "private" ] && [ $fell_on_sso_portal -eq 1 ] || return 2 - return 0 + local expected_to_fell_on_portal="" + [ "$install_type" == "private" ] && expected_to_fell_on_portal=1 || expected_to_fell_on_portal=0 + [ $fell_on_sso_portal -eq $expected_to_fell_on_portal ] && return 0 || return 0 } + + + #================================================= -# Unit tests +# The +# Actual +# Tests #================================================= + + + + +PACKAGE_LINTER () { + # Package linter + + start_test "Package linter" + + # Execute package linter and linter_result gets the return code of the package linter + ./package_linter/package_linter.py "$package_path" | tee -a "$complete_log" + ./package_linter/package_linter.py "$package_path" --json | tee -a "$complete_log" > $current_test_results + +# # Check we qualify for level 6, 7, 8 +# # Linter will have a warning called "app_in_github_org" if app ain't in the +# # yunohost-apps org... +# if ! cat "./temp_linter_result.json" | jq ".warning" | grep -q "app_in_github_org" +# then +# local pass_level_6="true" +# fi +# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_7" +# then +# local pass_level_7="true" +# fi +# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_8" +# then +# local pass_level_8="true" +# fi +# +# # If there are any critical errors, we'll force level 0 +# if [[ -n "$(cat "./temp_linter_result.json" | jq ".critical" | grep -v '\[\]')" ]] +# then +# local pass_level_0="false" +# # If there are any regular errors, we'll cap to 4 +# elif [[ -n "$(cat "./temp_linter_result.json" | jq ".error" | grep -v '\[\]')" ]] +# then +# local pass_level_4="false" +# # Otherwise, test pass (we'll display a warning depending on if there are +# # any remaning warnings or not) +# else +# if [[ -n "$(cat "./temp_linter_result.json" | jq ".warning" | grep -v '\[\]')" ]] +# then +# log_report_test_warning +# else +# log_report_test_success +# fi +# local pass_level_4="true" +# fi +} + TEST_INSTALL () { # Try to install in a sub path, on root or without url access # $1 = install type @@ -595,46 +739,47 @@ TEST_CHANGE_URL () { return $main_result } -# Define a function to split a file in multiple parts. Used for actions and config-panel toml -splitterAA() -{ - local bound="$1" - local file="$2" - - # If $2 is a real file - if [ -e "$file" ] - then - # Replace name of the file by its content - file="$(cat "$file")" - fi - - local file_lenght=$(echo "$file" | wc --lines | awk '{print $1}') - - bounds=($(echo "$file" | grep --line-number --extended-regexp "$bound" | cut -d':' -f1)) - - # Go for each line number (boundary) into the array - for line_number in $(seq 0 $(( ${#bounds[@]} -1 ))) - do - # The first bound is the next line number in the array - # That the low bound on which we cut - first_bound=$(( ${bounds[$line_number+1]} - 1 )) - # If there's no next cell in the array, we got -1, in such case, use the lenght of the file. - # We cut at the end of the file - test $first_bound -lt 0 && first_bound=$file_lenght - # The second bound is the current line number in the array minus the next one. - # The the upper bound in the file. - second_bound=$(( ${bounds[$line_number]} - $first_bound - 1 )) - # Cut the file a first time from the beginning to the first bound - # And a second time from the end, back to the second bound. - parts[line_number]="$(echo "$file" | head --lines=$first_bound \ - | tail --lines=$second_bound)" - done -} ACTIONS_CONFIG_PANEL () { - # Try the actions and config-panel features - + test_type=$1 + + # Define a function to split a file in multiple parts. Used for actions and config-panel toml + splitterAA() + { + local bound="$1" + local file="$2" + + # If $2 is a real file + if [ -e "$file" ] + then + # Replace name of the file by its content + file="$(cat "$file")" + fi + + local file_lenght=$(echo "$file" | wc --lines | awk '{print $1}') + + bounds=($(echo "$file" | grep --line-number --extended-regexp "$bound" | cut -d':' -f1)) + + # Go for each line number (boundary) into the array + for line_number in $(seq 0 $(( ${#bounds[@]} -1 ))) + do + # The first bound is the next line number in the array + # That the low bound on which we cut + first_bound=$(( ${bounds[$line_number+1]} - 1 )) + # If there's no next cell in the array, we got -1, in such case, use the lenght of the file. + # We cut at the end of the file + test $first_bound -lt 0 && first_bound=$file_lenght + # The second bound is the current line number in the array minus the next one. + # The the upper bound in the file. + second_bound=$(( ${bounds[$line_number]} - $first_bound - 1 )) + # Cut the file a first time from the beginning to the first bound + # And a second time from the end, back to the second bound. + parts[line_number]="$(echo "$file" | head --lines=$first_bound \ + | tail --lines=$second_bound)" + done + } + if [ "$test_type" == "actions" ] then start_test "Actions" @@ -884,222 +1029,3 @@ ACTIONS_CONFIG_PANEL () { return $main_result } -PACKAGE_LINTER () { - # Package linter - - start_test "Package linter" - - # Execute package linter and linter_result gets the return code of the package linter - "./package_linter/package_linter.py" "$package_path" | tee -a "$complete_log" - "./package_linter/package_linter.py" "$package_path" --json | tee -a "$complete_log" > $current_test_results - -# # Check we qualify for level 6, 7, 8 -# # Linter will have a warning called "app_in_github_org" if app ain't in the -# # yunohost-apps org... -# if ! cat "./temp_linter_result.json" | jq ".warning" | grep -q "app_in_github_org" -# then -# local pass_level_6="true" -# fi -# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_7" -# then -# local pass_level_7="true" -# fi -# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_8" -# then -# local pass_level_8="true" -# fi -# -# # If there are any critical errors, we'll force level 0 -# if [[ -n "$(cat "./temp_linter_result.json" | jq ".critical" | grep -v '\[\]')" ]] -# then -# local pass_level_0="false" -# # If there are any regular errors, we'll cap to 4 -# elif [[ -n "$(cat "./temp_linter_result.json" | jq ".error" | grep -v '\[\]')" ]] -# then -# local pass_level_4="false" -# # Otherwise, test pass (we'll display a warning depending on if there are -# # any remaning warnings or not) -# else -# if [[ -n "$(cat "./temp_linter_result.json" | jq ".warning" | grep -v '\[\]')" ]] -# then -# log_report_test_warning -# else -# log_report_test_success -# fi -# local pass_level_4="true" -# fi -} - -set_witness_files () { - # Create files to check if the remove script does not remove them accidentally - log_debug "Create witness files..." - - create_witness_file () { - [ "$2" = "file" ] && local action="touch" || local action="mkdir -p" - RUN_INSIDE_LXC $action $1 - } - - # Nginx conf - create_witness_file "/etc/nginx/conf.d/$DOMAIN.d/witnessfile.conf" file - create_witness_file "/etc/nginx/conf.d/$SUBDOMAIN.d/witnessfile.conf" file - - # /etc - create_witness_file "/etc/witnessfile" file - - # /opt directory - create_witness_file "/opt/witnessdir" directory - - # /var/www directory - create_witness_file "/var/www/witnessdir" directory - - # /home/yunohost.app/ - create_witness_file "/home/yunohost.app/witnessdir" directory - - # /var/log - create_witness_file "/var/log/witnessfile" file - - # Config fpm - create_witness_file "/etc/php/7.3/fpm/pool.d/witnessfile.conf" file - - # Config logrotate - create_witness_file "/etc/logrotate.d/witnessfile" file - - # Config systemd - create_witness_file "/etc/systemd/system/witnessfile.service" file - - # Database - local mysqlpwd=$(RUN_INSIDE_LXC cat /etc/yunohost/mysql) - RUN_INSIDE_LXC mysqladmin --user=root --password="$mysqlpwd" --wait status > /dev/null 2>&1 - echo "CREATE DATABASE witnessdb" | RUN_INSIDE_LXC mysql --user=root --password="$mysqlpwd" --wait > /dev/null 2>&1 -} - -check_witness_files () { - # Check all the witness files, to verify if them still here - - check_file_exist () { - if RUN_INSIDE_LXC test ! -e "$1" - then - log_error "The file $1 is missing ! Something gone wrong !" - SET_RESULT "failure" witness - fi - } - - # Nginx conf - check_file_exist "/etc/nginx/conf.d/$DOMAIN.d/witnessfile.conf" - check_file_exist "/etc/nginx/conf.d/$SUBDOMAIN.d/witnessfile.conf" - - # /etc - check_file_exist "/etc/witnessfile" - - # /opt directory - check_file_exist "/opt/witnessdir" - - # /var/www directory - check_file_exist "/var/www/witnessdir" - - # /home/yunohost.app/ - check_file_exist "/home/yunohost.app/witnessdir" - - # /var/log - check_file_exist "/var/log/witnessfile" - - # Config fpm - check_file_exist "/etc/php/7.3/fpm/pool.d/witnessfile.conf" - - # Config logrotate - check_file_exist "/etc/logrotate.d/witnessfile" - - # Config systemd - check_file_exist "/etc/systemd/system/witnessfile.service" - - # Database - local mysqlpwd=$(RUN_INSIDE_LXC cat /etc/yunohost/mysql) - if ! RUN_INSIDE_LXC mysqlshow --user=root --password="$mysqlpwd" witnessdb > /dev/null 2>&1 - then - log_error "The database witnessdb is missing ! Something gone wrong !" - SET_RESULT "failure" witness - return 1 - fi -} - - -RUN_ALL_TESTS() { - # Launch all tests successively - curl_error=0 - - # Be sure that the container is running - LXC_START "true" - - log_small_title "YunoHost versions" - - # Print the version of YunoHost from the LXC container - LXC_START "sudo yunohost --version" - - # Init the value for the current test - current_test_number=1 - - # The list of test contains for example "TEST_UPGRADE some_commit_id - for testfile in $(ls $TEST_CONTEXT/tests/*.json); - do - TEST_LAUNCHER $testfile - done - -} - -TEST_LAUNCHER () { - local testfile="$1" - - # Start the timer for this test - start_timer - # And keep this value separately - local global_start_timer=$starttime - - current_test_id=$(basename $testfile | cut -d. -f1) - current_test_infos="$TEST_CONTEXT/tests/$current_test_id.json" - current_test_results="$TEST_CONTEXT/results/$current_test_id.json" - echo "{}" > $current_test_results - - local test_type=$(jq -r '.test_type' $testfile) - local test_arg=$(jq -r '.test_arg' $testfile) - - # Execute the test - $test_type $test_arg - - [ $? -eq 0 ] && SET_RESULT "success" main_result || SET_RESULT "failure" main_result - - break_before_continue - - # Restore the started time for the timer - starttime=$global_start_timer - # End the timer for the test - stop_timer 2 - - LXC_STOP - - # Update the lock file with the date of the last finished test. - # $$ is the PID of package_check itself. - echo "$1 $2:$(date +%s):$$" > "$lock_file" -} - -SET_RESULT() { - local result=$1 - local name=$2 - [ "$result" == "success" ] && log_report_test_success || log_report_test_failed - local current_results="$(cat $current_test_results)" - echo "$current_results" | jq --arg result $result ".$name=\$result" > $current_test_results -} - -at_least_one_install_succeeded () { - - for TEST in $(ls $TEST_CONTEXT/tests/*.json) - do - local test_id=$(basename $TEST | cut -d. -f1) - jq -e '. | select(.test_type == "TEST_INSTALL")' $TEST >/dev/null \ - && jq -e '. | select(.main_result == "success")' $TEST_CONTEXT/results/$test_id.json >/dev/null \ - && return 0 - done - - log_error "All installs failed, therefore the following tests cannot be performed..." - return 1 -} - diff --git a/sub_scripts/witness.sh b/sub_scripts/witness.sh new file mode 100644 index 0000000..b458844 --- /dev/null +++ b/sub_scripts/witness.sh @@ -0,0 +1,92 @@ + +set_witness_files () { + # Create files to check if the remove script does not remove them accidentally + log_debug "Create witness files..." + + create_witness_file () { + [ "$2" = "file" ] && local action="touch" || local action="mkdir -p" + RUN_INSIDE_LXC $action $1 + } + + # Nginx conf + create_witness_file "/etc/nginx/conf.d/$DOMAIN.d/witnessfile.conf" file + create_witness_file "/etc/nginx/conf.d/$SUBDOMAIN.d/witnessfile.conf" file + + # /etc + create_witness_file "/etc/witnessfile" file + + # /opt directory + create_witness_file "/opt/witnessdir" directory + + # /var/www directory + create_witness_file "/var/www/witnessdir" directory + + # /home/yunohost.app/ + create_witness_file "/home/yunohost.app/witnessdir" directory + + # /var/log + create_witness_file "/var/log/witnessfile" file + + # Config fpm + create_witness_file "/etc/php/7.3/fpm/pool.d/witnessfile.conf" file + + # Config logrotate + create_witness_file "/etc/logrotate.d/witnessfile" file + + # Config systemd + create_witness_file "/etc/systemd/system/witnessfile.service" file + + # Database + local mysqlpwd=$(RUN_INSIDE_LXC cat /etc/yunohost/mysql) + RUN_INSIDE_LXC mysqladmin --user=root --password="$mysqlpwd" --wait status > /dev/null 2>&1 + echo "CREATE DATABASE witnessdb" | RUN_INSIDE_LXC mysql --user=root --password="$mysqlpwd" --wait > /dev/null 2>&1 +} + +check_witness_files () { + # Check all the witness files, to verify if them still here + + check_file_exist () { + if RUN_INSIDE_LXC test ! -e "$1" + then + log_error "The file $1 is missing ! Something gone wrong !" + SET_RESULT "failure" witness + fi + } + + # Nginx conf + check_file_exist "/etc/nginx/conf.d/$DOMAIN.d/witnessfile.conf" + check_file_exist "/etc/nginx/conf.d/$SUBDOMAIN.d/witnessfile.conf" + + # /etc + check_file_exist "/etc/witnessfile" + + # /opt directory + check_file_exist "/opt/witnessdir" + + # /var/www directory + check_file_exist "/var/www/witnessdir" + + # /home/yunohost.app/ + check_file_exist "/home/yunohost.app/witnessdir" + + # /var/log + check_file_exist "/var/log/witnessfile" + + # Config fpm + check_file_exist "/etc/php/7.3/fpm/pool.d/witnessfile.conf" + + # Config logrotate + check_file_exist "/etc/logrotate.d/witnessfile" + + # Config systemd + check_file_exist "/etc/systemd/system/witnessfile.service" + + # Database + local mysqlpwd=$(RUN_INSIDE_LXC cat /etc/yunohost/mysql) + if ! RUN_INSIDE_LXC mysqlshow --user=root --password="$mysqlpwd" witnessdb > /dev/null 2>&1 + then + log_error "The database witnessdb is missing ! Something gone wrong !" + SET_RESULT "failure" witness + return 1 + fi +}