From 4937a503ea87589c73fa6b7a829024e7f5b07fd9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 18 Dec 2020 01:48:37 +0100 Subject: [PATCH] Moar fixes + reimplement summary/level analysis --- lib/analyze_test_results.py | 491 ++++++++++++++++-------------------- lib/common.sh | 24 +- lib/tests.sh | 70 ++--- lib/tests_coordination.sh | 20 +- 4 files changed, 276 insertions(+), 329 deletions(-) diff --git a/lib/analyze_test_results.py b/lib/analyze_test_results.py index 3465172..4a72a13 100644 --- a/lib/analyze_test_results.py +++ b/lib/analyze_test_results.py @@ -1,284 +1,223 @@ +import sys +import json +import os -# Levels +def load_tests(test_folder): -#0 Broken -#1 Installable -#2 Installable in all situations -#3 Can be updated -#4 Backup and restore support -#5 Clean -#6 Open to contributions from the community -#7 Successfully pass all functional tests and linter tests -#8 Maintained and long-term good quality -#9 High quality app -#10 Package assessed as perfect + for test in sorted(os.listdir(test_folder + "/tests")): -# Linter stuff: + j = json.load(open(test_folder + "/tests/" + test)) + j["id"] = os.path.basename(test).split(".")[0] + j["results"] = json.load(open(test_folder + "/results/" + j["id"] + ".json")) + yield j -# # Check we qualify for level 6, 7, 8 -# # Linter will have a warning called "app_in_github_org" if app ain't in the -# # yunohost-apps org... -# if ! cat "./temp_linter_result.json" | jq ".warning" | grep -q "app_in_github_org" -# then -# local pass_level_6="true" -# fi -# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_7" -# then -# local pass_level_7="true" -# fi -# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_8" -# then -# local pass_level_8="true" -# fi -# -# # If there are any critical errors, we'll force level 0 -# if [[ -n "$(cat "./temp_linter_result.json" | jq ".critical" | grep -v '\[\]')" ]] -# then -# local pass_level_0="false" -# # If there are any regular errors, we'll cap to 4 -# elif [[ -n "$(cat "./temp_linter_result.json" | jq ".error" | grep -v '\[\]')" ]] -# then -# local pass_level_4="false" -# # Otherwise, test pass (we'll display a warning depending on if there are -# # any remaning warnings or not) -# else -# if [[ -n "$(cat "./temp_linter_result.json" | jq ".warning" | grep -v '\[\]')" ]] -# then -# log_report_test_warning -# else -# log_report_test_success -# fi -# local pass_level_4="true" -# fi +levels = [] + + +def level(level_, descr): + def decorator(f): + f.descr = descr + f.level = level_ + levels.insert(level_, f) + return f + return decorator + + +############################################################################### + + +@level(0, "Broken") +def level_0(tests): + return True + + +@level(1, "Installable in at least one scenario") +def level_1(tests): + """ + Any install test succeded + And there are no critical issues in the linter + """ + + linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"] + witness_missing_detected = any(t["results"].get("witness") for t in tests) + + return linter_tests != [] \ + and linter_tests[0]["results"]["critical"] == [] \ + and not witness_missing_detected \ + and any(t["results"]["main_result"] == "success" for t in install_tests) + + +@level(2, "Installable in all scenarios") +def level_2(tests): + """ + All install tests succeeded (and at least one test was made) + """ + + install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"] + + return install_tests != [] \ + and all(t["results"]["main_result"] == "success" for t in install_tests) + + +@level(3, "Can be upgraded") +def level_3(tests): + """ + All upgrade tests succeeded (and at least one test was made) + """ + + upgrade_tests = [t for t in tests if t["test_type"] == "TEST_UPGRADE"] + + return upgrade_tests != [] \ + and all(t["results"]["main_result"] == "success" for t in upgrade_tests) + + +@level(4, "Can be backup/restored") +def level_4(tests): + """ + All backup/restore tests succeded (and at least one test was made) + """ + + backup_tests = [t for t in tests if t["test_type"] == "TEST_BACKUP"] + + return backup_tests != [] \ + and all(t["results"]["main_result"] == "success" for t in backup_tests) + + +@level(5, "No linter errors") +def level_5(tests): + """ + Linter returned no errors (= main_result is success) + and no alias/path traversal issue detected during tests + """ + + alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests) + + linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + + return not alias_traversal_detected \ + and linter_tests != [] \ + and linter_tests[0]["results"]["main_result"] == "success" + + +@level(6, "App is in a community-operated git org") +def level_6(tests): + """ + The app is in the Yunohost-Apps organization + (the linter will report a warning named "is_in_github_org" if it's not) + """ + + linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + + return linter_tests != [] \ + and "is_in_github_org" not in linter_tests[0]["results"]["warning"] + + +@level(7, "Pass all tests + no linter warnings") +def level_7(tests): + """ + All tests succeeded + no warning in linter (that part is tested by the + # linter which will report a "qualify_for_level_7" in successes) + """ + + linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + + return all(t["results"]["main_result"] == "success" for t in tests) \ + and linter_tests != [] \ + and "qualify_for_level_7" in linter_tests[0]["results"]["success"] + + +@level(8, "Maintained and long-term good quality") +def level_8(tests): + """ + App is maintained and long-term good quality (this is tested by the linter + which will report a "qualify_for_level_8") + """ + + linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + + return linter_tests != [] \ + and "qualify_for_level_8" in linter_tests[0]["results"]["success"] + + +@level(9, "Flagged high-quality in app catalog") +def level_9(tests): + """ + App is flagged high-quality in the app catalog (this is tested by the linter + which will rpeort a "qualify_for_level_9") + """ + linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + + return linter_tests != [] \ + and "qualify_for_level_9" in linter_tests[0]["results"]["success"] + + +tests = list(load_tests(sys.argv[1])) + +test_types = { + "PACKAGE_LINTER": "Package linter", + "TEST_INSTALL": "Install", + "TEST_UPGRADE": "Upgrade", + "TEST_BACKUP_RESTORE": "Backup/restore", + "TEST_CHANGE_URL": "Change url", + "TEST_PORT_ALREADY_USED": "Port already used", + "ACTIONS_CONFIG_PANEL": "Config/panel" +} + +OK = ' \033[1m\033[92mOK\033[0m ' +FAIL = '\033[91mfail\033[0m' + +print("Tests summary") +print("=============") +for test in tests: + test_display_name = test_types[test["test_type"]] + if test["test_arg"]: + test_display_name += " (%s)" % test["test_arg"][:8] + test_display_name += ":" + + result = OK if test["results"]["main_result"] == "success" else FAIL + print("{test: <25}{result}".format(test=test_display_name, result=result)) + +print() +print("Level results") +print("=============") + +stop_global_level_bump = False + +global_level = level_0 + +for level in levels[1:]: + level.passed = level(tests) + + if not level.passed: + stop_global_level_bump = True + + if not stop_global_level_bump: + global_level = level + display = OK + else: + display = " ok " if level.passed else "" + + print("Level {i} {descr: <40} {result}".format(i=level.level, + descr="(%s)"%level.descr[:38], result=display)) + +print() +print("\033[1mGlobal level for this application: %s (%s)\033[0m" % (global_level.level, global_level.descr)) +print() + + +summary = { + "tests": [{ + "test_type": t["test_type"], + "test_arg": t["test_arg"], + "main_result": t["results"]["main_result"] + } for t in tests], + "levels": {level.level: level.passed for level in levels[1:]}, + "global_level": global_level.level +} + +sys.stderr.write(json.dumps(summary)) - - - - - - - - - - -# -# local test_serie_id=$1 -# source $TEST_CONTEXT/$test_serie_id/results -# -# # Print the test result -# print_result () { -# -# # Print the result of this test -# # we use printf to force the length to 30 (filled with space) -# testname=$(printf %-30.30s "$1:") -# if [ $2 -eq 1 ] -# then -# echo "$testname ${BOLD}${GREEN}SUCCESS${NORMAL}" -# elif [ $2 -eq -1 ] -# then -# echo "$testname ${BOLD}${RED}FAIL${NORMAL}" -# else -# echo "$testname Not evaluated." -# fi -# } -# -# # Print the result for each test -# echo -e "\n\n" -# print_result "Package linter" $RESULT_linter -# print_result "Install (root)" $RESULT_check_root -# print_result "Install (subpath)" $RESULT_check_subdir -# print_result "Install (no url)" $RESULT_check_nourl -# print_result "Install (private)" $RESULT_check_private -# print_result "Install (multi-instance)" $RESULT_check_multi_instance -# print_result "Upgrade" $RESULT_check_upgrade -# print_result "Backup" $RESULT_check_backup -# print_result "Restore" $RESULT_check_restore -# print_result "Change URL" $RESULT_change_url -# print_result "Port already used" $RESULT_check_port -# print_result "Actions and config-panel" $RESULT_action_config_panel -# -# # Determine the level for this app -# -# # Each level can has 5 different values -# # 0 -> If this level can't be validated -# # 1 -> If this level is forced. Even if the tests fails -# # 2 -> Indicates the tests had previously validated this level -# # auto -> This level has not a value yet. -# # na -> This level will not be checked, but it'll be ignored in the final sum -# -# # Set default values for level, if they're empty. -# test -n "${level[1]}" || level[1]=auto -# test -n "${level[2]}" || level[2]=auto -# test -n "${level[3]}" || level[3]=auto -# test -n "${level[4]}" || level[4]=auto -# test -n "${level[5]}" || level[5]=auto -# test -n "${level[5]}" || level[5]=auto -# test -n "${level[6]}" || level[6]=auto -# test -n "${level[7]}" || level[7]=auto -# test -n "${level[8]}" || level[8]=auto -# test -n "${level[9]}" || level[9]=0 -# test -n "${level[10]}" || level[10]=0 -# -# pass_level_1() { -# # FIXME FIXME #FIXME -# return 0 -# } -# -# pass_level_2() { -# # -> The package can be install and remove in all tested configurations. -# # Validated if none install failed -# [ $RESULT_check_subdir -ne -1 ] && \ -# [ $RESULT_check_root -ne -1 ] && \ -# [ $RESULT_check_private -ne -1 ] && \ -# [ $RESULT_check_multi_instance -ne -1 ] -# } -# -# pass_level_3() { -# # -> The package can be upgraded from the same version. -# # Validated if the upgrade is ok. Or if the upgrade has been not tested but already validated before. -# [ $RESULT_check_upgrade -eq 1 ] || \ -# ( [ $RESULT_check_upgrade -ne -1 ] && \ -# [ "${level[3]}" == "2" ] ) -# } -# -# pass_level_4() { -# # -> The package can be backup and restore without error -# # Validated if backup and restore are ok. Or if backup and restore have been not tested but already validated before. -# ( [ $RESULT_check_backup -eq 1 ] && \ -# [ $RESULT_check_restore -eq 1 ] ) || \ -# ( [ $RESULT_check_backup -ne -1 ] && \ -# [ $RESULT_check_restore -ne -1 ] && \ -# [ "${level[4]}" == "2" ] ) -# } -# -# pass_level_5() { -# # -> The package have no error with package linter -# # -> The package does not have any alias_traversal error -# # Validated if Linter is ok. Or if Linter has been not tested but already validated before. -# [ $RESULT_alias_traversal -ne 1 ] && \ -# ([ $RESULT_linter -ge 1 ] || \ -# ( [ $RESULT_linter -eq 0 ] && \ -# [ "${level[5]}" == "2" ] ) ) -# } -# -# pass_level_6() { -# # -> The package can be backup and restore without error -# # This is from the linter, tests if app is the Yunohost-apps organization -# [ $RESULT_linter_level_6 -eq 1 ] || \ -# ([ $RESULT_linter_level_6 -eq 0 ] && \ -# [ "${level[6]}" == "2" ] ) -# } -# -# pass_level_7() { -# # -> None errors in all tests performed -# # Validated if none errors is happened. -# [ $RESULT_check_subdir -ne -1 ] && \ -# [ $RESULT_check_upgrade -ne -1 ] && \ -# [ $RESULT_check_private -ne -1 ] && \ -# [ $RESULT_check_multi_instance -ne -1 ] && \ -# [ $RESULT_check_port -ne -1 ] && \ -# [ $RESULT_check_backup -ne -1 ] && \ -# [ $RESULT_check_restore -ne -1 ] && \ -# [ $RESULT_change_url -ne -1 ] && \ -# [ $RESULT_action_config_panel -ne -1 ] && \ -# ([ $RESULT_linter_level_7 -ge 1 ] || -# ([ $RESULT_linter_level_7 -eq 0 ] && \ -# [ "${level[8]}" == "2" ] )) -# } -# -# pass_level_8() { -# # This happens in the linter -# # When writing this, defined as app being maintained + long term quality (= -# # certain amount of time level 5+ in the last year) -# [ $RESULT_linter_level_8 -ge 1 ] || \ -# ([ $RESULT_linter_level_8 -eq 0 ] && \ -# [ "${level[8]}" == "2" ] ) -# } -# -# pass_level_9() { -# list_url="https://raw.githubusercontent.com/YunoHost/apps/master/apps.json" -# curl --silent $list_url | jq ".[\"$app_id\"].high_quality" | grep -q "true" -# } -# -# # Check if the level can be changed -# level_can_change () { -# # If the level is set at auto, it's waiting for a change -# # And if it's set at 2, its value can be modified by a new result -# [ "${level[$1]}" == "auto" ] || [ "${level[$1]}" -eq 2 ] -# } -# -# if level_can_change 1; then pass_level_1 && level[1]=2 || level[1]=0; fi -# if level_can_change 2; then pass_level_2 && level[2]=2 || level[2]=0; fi -# if level_can_change 3; then pass_level_3 && level[3]=2 || level[3]=0; fi -# if level_can_change 4; then pass_level_4 && level[4]=2 || level[4]=0; fi -# if level_can_change 5; then pass_level_5 && level[5]=2 || level[5]=0; fi -# if level_can_change 6; then pass_level_6 && level[6]=2 || level[6]=0; fi -# if level_can_change 7; then pass_level_7 && level[7]=2 || level[7]=0; fi -# if level_can_change 8; then pass_level_8 && level[8]=2 || level[8]=0; fi -# if level_can_change 9; then pass_level_9 && level[9]=2 || level[9]=0; fi -# -# # Level 10 has no definition yet -# level[10]=0 -# -# # Initialize the global level -# global_level=0 -# -# # Calculate the final level -# for i in `seq 1 10` -# do -# -# # If there is a level still at 'auto', it's a mistake. -# if [ "${level[i]}" == "auto" ] -# then -# # So this level will set at 0. -# level[i]=0 -# -# # If the level is at 1 or 2. The global level will be set at this level -# elif [ "${level[i]}" -ge 1 ] -# then -# global_level=$i -# -# # But, if the level is at 0, the loop stop here -# # Like that, the global level rise while none level have failed -# else -# break -# fi -# done -# -# # If some witness files was missing, it's a big error ! So, the level fall immediately at 0. -# if [ $RESULT_witness -eq 1 ] -# then -# log_error "Some witness files has been deleted during those tests ! It's a very bad thing !" -# global_level=0 -# fi -# -# # If the package linter returned a critical error, the app is flagged as broken / level 0 -# if [ $RESULT_linter_broken -eq 1 ] -# then -# log_error "The package linter reported a critical failure ! App is considered broken !" -# global_level=0 -# fi -# -# if [ $RESULT_alias_traversal -eq 1 ] -# then -# log_error "Issue alias_traversal was detected ! Please see here https://github.com/YunoHost/example_ynh/pull/45 to fix that." -# fi -# -# # Then, print the levels -# # Print the global level -# verbose_level=$(grep "^$global_level " "./levels.list" | cut -c4-) -# -# log_info "Level of this application: $global_level ($verbose_level)" -# -# # And print the value for each level -# for i in `seq 1 10` -# do -# display="0" -# if [ "${level[$i]}" -ge 1 ]; then -# display="1" -# fi -# echo -e "\t Level $i: $display" -# done diff --git a/lib/common.sh b/lib/common.sh index 297fdc2..c68b4b1 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -27,7 +27,7 @@ clean_exit () { LXC_RESET - [ -n "$TEST_CONTEXT" ] rm -rf "$TEST_CONTEXT" + [ -n "$TEST_CONTEXT" ] && rm -rf "$TEST_CONTEXT" rm -f "$lock_file" exit $1 @@ -78,7 +78,7 @@ readonly WHITE=$(printf '\033[39m') function log_title() { - cat << EOF + cat << EOF | tee -a /proc/self/fd/3 ${BOLD} =================================== $1 @@ -89,51 +89,51 @@ EOF function log_small_title() { - echo -e "\n${BOLD} > ${1}${NORMAL}\n" + echo -e "\n${BOLD} > ${1}${NORMAL}\n" | tee -a /proc/self/fd/3 } function log_debug() { - echo "$1" >&3 + echo "$1" >> /proc/self/fd/3 } function log_info() { - echo "${1}" + echo "${1}" | tee -a /proc/self/fd/3 } function log_success() { - echo "${BOLD}${GREEN}Success: ${1}${NORMAL}" + echo "${BOLD}${GREEN}Success: ${1}${NORMAL}" | tee -a /proc/self/fd/3 } function log_warning() { - echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}" + echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}" | tee -a /proc/self/fd/3 } function log_error() { - echo "${BOLD}${RED}Error: ${1}${NORMAL}" + echo "${BOLD}${RED}Error: ${1}${NORMAL}" | tee -a /proc/self/fd/3 } function log_critical() { - echo "${BOLD}${RED}Critical: ${1}${NORMAL}" + echo "${BOLD}${RED}Critical: ${1}${NORMAL}" | tee -a /proc/self/fd/3 clean_exit 1 } function log_report_test_success () { - echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n" + echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n" | tee -a /proc/self/fd/3 } function log_report_test_warning () { - echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n" + echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n" | tee -a /proc/self/fd/3 } function log_report_test_failed () { - echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n" + echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n" | tee -a /proc/self/fd/3 } #================================================= diff --git a/lib/tests.sh b/lib/tests.sh index e80127c..b99d327 100644 --- a/lib/tests.sh +++ b/lib/tests.sh @@ -1,7 +1,7 @@ #!/bin/bash #================================================= -# Logistic helpers +# "Low-level" logistic helpers #================================================= _RUN_YUNOHOST_CMD() { @@ -258,19 +258,24 @@ PACKAGE_LINTER () { # Execute package linter and linter_result gets the return code of the package linter ./package_linter/package_linter.py "$package_path" | tee -a "$complete_log" ./package_linter/package_linter.py "$package_path" --json | tee -a "$complete_log" > $current_test_results + + return $? } TEST_INSTALL () { - # Try to install in a sub path, on root or without url access - # $1 = install type local install_type=$1 + + # This is a separate case ... at least from an hystorical point of view ... + # but it helpers for semantic that the test is a "TEST_INSTALL" ... + [ "$install_type" = "multi" ] && { _TEST_MULTI_INSTANCE; return $?; } + local check_path="/" local is_public="1" - [ "$install_type" = "subdir" ] && { start_test "Installation in a sub path"; local check_path=/path; } - [ "$install_type" = "root" ] && { start_test "Installation on the root"; } - [ "$install_type" = "nourl" ] && { start_test "Installation without url access"; local check_path=""; } - [ "$install_type" = "private" ] && { start_test "Installation in private mode"; local is_public="0"; } + [ "$install_type" = "subdir" ] && { start_test "Installation in a sub path"; local check_path=/path; } + [ "$install_type" = "root" ] && { start_test "Installation on the root"; } + [ "$install_type" = "nourl" ] && { start_test "Installation without url access"; local check_path=""; } + [ "$install_type" = "private" ] && { start_test "Installation in private mode"; local is_public="0"; } local snapname=snap_${install_type}install LOAD_LXC_SNAPSHOT snap0 @@ -298,6 +303,29 @@ TEST_INSTALL () { return $? } +_TEST_MULTI_INSTANCE () { + + start_test "Multi-instance installations" + + # Check if an install have previously work + at_least_one_install_succeeded || return 1 + + local check_path=$(default_install_path) + + LOAD_LXC_SNAPSHOT snap0 + + log_small_title "First installation: path=$DOMAIN$check_path" \ + &&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \ + && log_small_title "Second installation: path=$SUBDOMAIN$check_path" \ + && _INSTALL_APP "path=$check_path" \ + && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \ + && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2 \ + && _REMOVE_APP \ + && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2 + + return $? +} + TEST_UPGRADE () { local commit=$1 @@ -351,27 +379,6 @@ TEST_UPGRADE () { return $? } -TEST_MULTI_INSTANCE () { - - start_test "Multi-instance installations" - - # Check if an install have previously work - at_least_one_install_succeeded || return 1 - - local check_path=$(default_install_path) - - LOAD_LXC_SNAPSHOT snap0 - - log_small_title "First installation: path=$DOMAIN$check_path" \ - &&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \ - && log_small_title "Second installation: path=$SUBDOMAIN$check_path" \ - &&_INSTALL_APP "path=$check_path" \ - && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \ - && _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2 - - return $? -} - TEST_PORT_ALREADY_USED () { start_test "Port already used" @@ -464,6 +471,7 @@ TEST_BACKUP_RESTORE () { # Place the copy of the backup archive in the container. sudo lxc file push -r ./ynh_backups $LXC_NAME/home/yunohost.backup/archives/ + RUN_INSIDE_LXC ls -l /home/yunohost.backup/archives/ log_small_title "Restore on a clean YunoHost system..." fi @@ -489,10 +497,10 @@ TEST_CHANGE_URL () { # Check if an install have previously work at_least_one_install_succeeded || return 1 this_is_a_web_app || return 0 - + log_small_title "Preliminary install..." \ && _LOAD_SNAPSHOT_OR_INSTALL_APP "/" - + local ret=$? [ $ret -eq 0 ] || { return 1; } @@ -548,7 +556,7 @@ TEST_CHANGE_URL () { ACTIONS_CONFIG_PANEL () { - + test_type=$1 # Define a function to split a file in multiple parts. Used for actions and config-panel toml diff --git a/lib/tests_coordination.sh b/lib/tests_coordination.sh index d6306fa..0e9cab2 100644 --- a/lib/tests_coordination.sh +++ b/lib/tests_coordination.sh @@ -7,7 +7,7 @@ source lib/witness.sh complete_log="./Complete.log" # Purge some log files -> "$complete_log" +rm -f "$complete_log" && touch "$complete_log" # Redirect fd 3 (=debug steam) to complete log exec 3>>$complete_log @@ -16,9 +16,6 @@ exec 3>>$complete_log # Parse the check_process and generate jsons that describe tests to run #======================================================================= -# Parse the check_process only if it's exist -check_process="$package_path/check_process" - # Extract a section found between $1 and $2 from the file $3 extract_check_process_section () { local source_file="${3:-$check_process}" @@ -156,11 +153,11 @@ parse_check_process() { fi is_test_enabled pkg_linter && add_test "PACKAGE_LINTER" - is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir" is_test_enabled setup_root && add_test "TEST_INSTALL" "root" + is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir" is_test_enabled setup_nourl && add_test "TEST_INSTALL" "nourl" is_test_enabled setup_private && add_test "TEST_INSTALL" "private" - is_test_enabled multi_instance && add_test "TEST_MULTI_INSTANCE" + is_test_enabled multi_instance && add_test "TEST_INSTALL" "multi" is_test_enabled backup_restore && add_test "TEST_BACKUP_RESTORE" # Upgrades @@ -220,15 +217,15 @@ guess_test_configuration() { local install_args=$(python "./lib/manifest_parsing.py" "$package_path/manifest.json" | cut -d ':' -f1,2 | tr ':' '=' | tr '\n' '&') add_test "PACKAGE_LINTER" - add_test "TEST_INSTALL subdir" - add_test "TEST_INSTALL root" + add_test "TEST_INSTALL" "root" + add_test "TEST_INSTALL" "subdir" if echo $install_args | grep -q "is_public=" then add_test "TEST_INSTALL" "private" fi if grep multi_instance "$package_path/manifest.json" | grep -q true then - add_test "TEST_MULTI_INSTANCE" + add_test "TEST_INSTALL" "multi" fi add_test "TEST_BACKUP_RESTORE" add_test "TEST_UPGRADE" @@ -243,6 +240,9 @@ run_all_tests() { mkdir -p $TEST_CONTEXT/tests mkdir -p $TEST_CONTEXT/results + # Parse the check_process only if it's exist + check_process="$package_path/check_process" + [ -e "$check_process" ] \ && parse_check_process \ || guess_test_configuration @@ -281,7 +281,7 @@ run_all_tests() { done # Print the final results of the tests - # FIXME COMPUTE_RESULTS_SUMMARY $test_serie_id + python3 lib/analyze_test_results.py $TEST_CONTEXT 2>$TEST_CONTEXT/summary.json # Restore the started time for the timer starttime=$complete_start_timer