mirror of
https://github.com/YunoHost/package_check.git
synced 2024-09-03 20:06:20 +02:00
Moar fixes + reimplement summary/level analysis
This commit is contained in:
Alexandre Aubin
parent
0415e0fcf5
commit
4937a503ea
4 changed files with 276 additions and 329 deletions
|
|
@ -1,284 +1,223 @@
|
|||
import sys
|
||||
import json
|
||||
import os
|
||||
|
||||
# Levels
|
||||
def load_tests(test_folder):
|
||||
|
||||
#0 Broken
|
||||
#1 Installable
|
||||
#2 Installable in all situations
|
||||
#3 Can be updated
|
||||
#4 Backup and restore support
|
||||
#5 Clean
|
||||
#6 Open to contributions from the community
|
||||
#7 Successfully pass all functional tests and linter tests
|
||||
#8 Maintained and long-term good quality
|
||||
#9 High quality app
|
||||
#10 Package assessed as perfect
|
||||
for test in sorted(os.listdir(test_folder + "/tests")):
|
||||
|
||||
# Linter stuff:
|
||||
j = json.load(open(test_folder + "/tests/" + test))
|
||||
j["id"] = os.path.basename(test).split(".")[0]
|
||||
j["results"] = json.load(open(test_folder + "/results/" + j["id"] + ".json"))
|
||||
yield j
|
||||
|
||||
|
||||
# # Check we qualify for level 6, 7, 8
|
||||
# # Linter will have a warning called "app_in_github_org" if app ain't in the
|
||||
# # yunohost-apps org...
|
||||
# if ! cat "./temp_linter_result.json" | jq ".warning" | grep -q "app_in_github_org"
|
||||
# then
|
||||
# local pass_level_6="true"
|
||||
# fi
|
||||
# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_7"
|
||||
# then
|
||||
# local pass_level_7="true"
|
||||
# fi
|
||||
# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_8"
|
||||
# then
|
||||
# local pass_level_8="true"
|
||||
# fi
|
||||
#
|
||||
# # If there are any critical errors, we'll force level 0
|
||||
# if [[ -n "$(cat "./temp_linter_result.json" | jq ".critical" | grep -v '\[\]')" ]]
|
||||
# then
|
||||
# local pass_level_0="false"
|
||||
# # If there are any regular errors, we'll cap to 4
|
||||
# elif [[ -n "$(cat "./temp_linter_result.json" | jq ".error" | grep -v '\[\]')" ]]
|
||||
# then
|
||||
# local pass_level_4="false"
|
||||
# # Otherwise, test pass (we'll display a warning depending on if there are
|
||||
# # any remaning warnings or not)
|
||||
# else
|
||||
# if [[ -n "$(cat "./temp_linter_result.json" | jq ".warning" | grep -v '\[\]')" ]]
|
||||
# then
|
||||
# log_report_test_warning
|
||||
# else
|
||||
# log_report_test_success
|
||||
# fi
|
||||
# local pass_level_4="true"
|
||||
# fi
|
||||
levels = []
|
||||
|
||||
|
||||
def level(level_, descr):
|
||||
def decorator(f):
|
||||
f.descr = descr
|
||||
f.level = level_
|
||||
levels.insert(level_, f)
|
||||
return f
|
||||
return decorator
|
||||
|
||||
|
||||
###############################################################################
|
||||
|
||||
|
||||
@level(0, "Broken")
|
||||
def level_0(tests):
|
||||
return True
|
||||
|
||||
|
||||
@level(1, "Installable in at least one scenario")
|
||||
def level_1(tests):
|
||||
"""
|
||||
Any install test succeded
|
||||
And there are no critical issues in the linter
|
||||
"""
|
||||
|
||||
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||
install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
|
||||
witness_missing_detected = any(t["results"].get("witness") for t in tests)
|
||||
|
||||
return linter_tests != [] \
|
||||
and linter_tests[0]["results"]["critical"] == [] \
|
||||
and not witness_missing_detected \
|
||||
and any(t["results"]["main_result"] == "success" for t in install_tests)
|
||||
|
||||
|
||||
@level(2, "Installable in all scenarios")
|
||||
def level_2(tests):
|
||||
"""
|
||||
All install tests succeeded (and at least one test was made)
|
||||
"""
|
||||
|
||||
install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
|
||||
|
||||
return install_tests != [] \
|
||||
and all(t["results"]["main_result"] == "success" for t in install_tests)
|
||||
|
||||
|
||||
@level(3, "Can be upgraded")
|
||||
def level_3(tests):
|
||||
"""
|
||||
All upgrade tests succeeded (and at least one test was made)
|
||||
"""
|
||||
|
||||
upgrade_tests = [t for t in tests if t["test_type"] == "TEST_UPGRADE"]
|
||||
|
||||
return upgrade_tests != [] \
|
||||
and all(t["results"]["main_result"] == "success" for t in upgrade_tests)
|
||||
|
||||
|
||||
@level(4, "Can be backup/restored")
|
||||
def level_4(tests):
|
||||
"""
|
||||
All backup/restore tests succeded (and at least one test was made)
|
||||
"""
|
||||
|
||||
backup_tests = [t for t in tests if t["test_type"] == "TEST_BACKUP"]
|
||||
|
||||
return backup_tests != [] \
|
||||
and all(t["results"]["main_result"] == "success" for t in backup_tests)
|
||||
|
||||
|
||||
@level(5, "No linter errors")
|
||||
def level_5(tests):
|
||||
"""
|
||||
Linter returned no errors (= main_result is success)
|
||||
and no alias/path traversal issue detected during tests
|
||||
"""
|
||||
|
||||
alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests)
|
||||
|
||||
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||
|
||||
return not alias_traversal_detected \
|
||||
and linter_tests != [] \
|
||||
and linter_tests[0]["results"]["main_result"] == "success"
|
||||
|
||||
|
||||
@level(6, "App is in a community-operated git org")
|
||||
def level_6(tests):
|
||||
"""
|
||||
The app is in the Yunohost-Apps organization
|
||||
(the linter will report a warning named "is_in_github_org" if it's not)
|
||||
"""
|
||||
|
||||
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||
|
||||
return linter_tests != [] \
|
||||
and "is_in_github_org" not in linter_tests[0]["results"]["warning"]
|
||||
|
||||
|
||||
@level(7, "Pass all tests + no linter warnings")
|
||||
def level_7(tests):
|
||||
"""
|
||||
All tests succeeded + no warning in linter (that part is tested by the
|
||||
# linter which will report a "qualify_for_level_7" in successes)
|
||||
"""
|
||||
|
||||
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||
|
||||
return all(t["results"]["main_result"] == "success" for t in tests) \
|
||||
and linter_tests != [] \
|
||||
and "qualify_for_level_7" in linter_tests[0]["results"]["success"]
|
||||
|
||||
|
||||
@level(8, "Maintained and long-term good quality")
|
||||
def level_8(tests):
|
||||
"""
|
||||
App is maintained and long-term good quality (this is tested by the linter
|
||||
which will report a "qualify_for_level_8")
|
||||
"""
|
||||
|
||||
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||
|
||||
return linter_tests != [] \
|
||||
and "qualify_for_level_8" in linter_tests[0]["results"]["success"]
|
||||
|
||||
|
||||
@level(9, "Flagged high-quality in app catalog")
|
||||
def level_9(tests):
|
||||
"""
|
||||
App is flagged high-quality in the app catalog (this is tested by the linter
|
||||
which will rpeort a "qualify_for_level_9")
|
||||
"""
|
||||
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||
|
||||
return linter_tests != [] \
|
||||
and "qualify_for_level_9" in linter_tests[0]["results"]["success"]
|
||||
|
||||
|
||||
tests = list(load_tests(sys.argv[1]))
|
||||
|
||||
test_types = {
|
||||
"PACKAGE_LINTER": "Package linter",
|
||||
"TEST_INSTALL": "Install",
|
||||
"TEST_UPGRADE": "Upgrade",
|
||||
"TEST_BACKUP_RESTORE": "Backup/restore",
|
||||
"TEST_CHANGE_URL": "Change url",
|
||||
"TEST_PORT_ALREADY_USED": "Port already used",
|
||||
"ACTIONS_CONFIG_PANEL": "Config/panel"
|
||||
}
|
||||
|
||||
OK = ' \033[1m\033[92mOK\033[0m '
|
||||
FAIL = '\033[91mfail\033[0m'
|
||||
|
||||
print("Tests summary")
|
||||
print("=============")
|
||||
for test in tests:
|
||||
test_display_name = test_types[test["test_type"]]
|
||||
if test["test_arg"]:
|
||||
test_display_name += " (%s)" % test["test_arg"][:8]
|
||||
test_display_name += ":"
|
||||
|
||||
result = OK if test["results"]["main_result"] == "success" else FAIL
|
||||
print("{test: <25}{result}".format(test=test_display_name, result=result))
|
||||
|
||||
print()
|
||||
print("Level results")
|
||||
print("=============")
|
||||
|
||||
stop_global_level_bump = False
|
||||
|
||||
global_level = level_0
|
||||
|
||||
for level in levels[1:]:
|
||||
level.passed = level(tests)
|
||||
|
||||
if not level.passed:
|
||||
stop_global_level_bump = True
|
||||
|
||||
if not stop_global_level_bump:
|
||||
global_level = level
|
||||
display = OK
|
||||
else:
|
||||
display = " ok " if level.passed else ""
|
||||
|
||||
print("Level {i} {descr: <40} {result}".format(i=level.level,
|
||||
descr="(%s)"%level.descr[:38], result=display))
|
||||
|
||||
print()
|
||||
print("\033[1mGlobal level for this application: %s (%s)\033[0m" % (global_level.level, global_level.descr))
|
||||
print()
|
||||
|
||||
|
||||
summary = {
|
||||
"tests": [{
|
||||
"test_type": t["test_type"],
|
||||
"test_arg": t["test_arg"],
|
||||
"main_result": t["results"]["main_result"]
|
||||
} for t in tests],
|
||||
"levels": {level.level: level.passed for level in levels[1:]},
|
||||
"global_level": global_level.level
|
||||
}
|
||||
|
||||
sys.stderr.write(json.dumps(summary))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#
|
||||
# local test_serie_id=$1
|
||||
# source $TEST_CONTEXT/$test_serie_id/results
|
||||
#
|
||||
# # Print the test result
|
||||
# print_result () {
|
||||
#
|
||||
# # Print the result of this test
|
||||
# # we use printf to force the length to 30 (filled with space)
|
||||
# testname=$(printf %-30.30s "$1:")
|
||||
# if [ $2 -eq 1 ]
|
||||
# then
|
||||
# echo "$testname ${BOLD}${GREEN}SUCCESS${NORMAL}"
|
||||
# elif [ $2 -eq -1 ]
|
||||
# then
|
||||
# echo "$testname ${BOLD}${RED}FAIL${NORMAL}"
|
||||
# else
|
||||
# echo "$testname Not evaluated."
|
||||
# fi
|
||||
# }
|
||||
#
|
||||
# # Print the result for each test
|
||||
# echo -e "\n\n"
|
||||
# print_result "Package linter" $RESULT_linter
|
||||
# print_result "Install (root)" $RESULT_check_root
|
||||
# print_result "Install (subpath)" $RESULT_check_subdir
|
||||
# print_result "Install (no url)" $RESULT_check_nourl
|
||||
# print_result "Install (private)" $RESULT_check_private
|
||||
# print_result "Install (multi-instance)" $RESULT_check_multi_instance
|
||||
# print_result "Upgrade" $RESULT_check_upgrade
|
||||
# print_result "Backup" $RESULT_check_backup
|
||||
# print_result "Restore" $RESULT_check_restore
|
||||
# print_result "Change URL" $RESULT_change_url
|
||||
# print_result "Port already used" $RESULT_check_port
|
||||
# print_result "Actions and config-panel" $RESULT_action_config_panel
|
||||
#
|
||||
# # Determine the level for this app
|
||||
#
|
||||
# # Each level can has 5 different values
|
||||
# # 0 -> If this level can't be validated
|
||||
# # 1 -> If this level is forced. Even if the tests fails
|
||||
# # 2 -> Indicates the tests had previously validated this level
|
||||
# # auto -> This level has not a value yet.
|
||||
# # na -> This level will not be checked, but it'll be ignored in the final sum
|
||||
#
|
||||
# # Set default values for level, if they're empty.
|
||||
# test -n "${level[1]}" || level[1]=auto
|
||||
# test -n "${level[2]}" || level[2]=auto
|
||||
# test -n "${level[3]}" || level[3]=auto
|
||||
# test -n "${level[4]}" || level[4]=auto
|
||||
# test -n "${level[5]}" || level[5]=auto
|
||||
# test -n "${level[5]}" || level[5]=auto
|
||||
# test -n "${level[6]}" || level[6]=auto
|
||||
# test -n "${level[7]}" || level[7]=auto
|
||||
# test -n "${level[8]}" || level[8]=auto
|
||||
# test -n "${level[9]}" || level[9]=0
|
||||
# test -n "${level[10]}" || level[10]=0
|
||||
#
|
||||
# pass_level_1() {
|
||||
# # FIXME FIXME #FIXME
|
||||
# return 0
|
||||
# }
|
||||
#
|
||||
# pass_level_2() {
|
||||
# # -> The package can be install and remove in all tested configurations.
|
||||
# # Validated if none install failed
|
||||
# [ $RESULT_check_subdir -ne -1 ] && \
|
||||
# [ $RESULT_check_root -ne -1 ] && \
|
||||
# [ $RESULT_check_private -ne -1 ] && \
|
||||
# [ $RESULT_check_multi_instance -ne -1 ]
|
||||
# }
|
||||
#
|
||||
# pass_level_3() {
|
||||
# # -> The package can be upgraded from the same version.
|
||||
# # Validated if the upgrade is ok. Or if the upgrade has been not tested but already validated before.
|
||||
# [ $RESULT_check_upgrade -eq 1 ] || \
|
||||
# ( [ $RESULT_check_upgrade -ne -1 ] && \
|
||||
# [ "${level[3]}" == "2" ] )
|
||||
# }
|
||||
#
|
||||
# pass_level_4() {
|
||||
# # -> The package can be backup and restore without error
|
||||
# # Validated if backup and restore are ok. Or if backup and restore have been not tested but already validated before.
|
||||
# ( [ $RESULT_check_backup -eq 1 ] && \
|
||||
# [ $RESULT_check_restore -eq 1 ] ) || \
|
||||
# ( [ $RESULT_check_backup -ne -1 ] && \
|
||||
# [ $RESULT_check_restore -ne -1 ] && \
|
||||
# [ "${level[4]}" == "2" ] )
|
||||
# }
|
||||
#
|
||||
# pass_level_5() {
|
||||
# # -> The package have no error with package linter
|
||||
# # -> The package does not have any alias_traversal error
|
||||
# # Validated if Linter is ok. Or if Linter has been not tested but already validated before.
|
||||
# [ $RESULT_alias_traversal -ne 1 ] && \
|
||||
# ([ $RESULT_linter -ge 1 ] || \
|
||||
# ( [ $RESULT_linter -eq 0 ] && \
|
||||
# [ "${level[5]}" == "2" ] ) )
|
||||
# }
|
||||
#
|
||||
# pass_level_6() {
|
||||
# # -> The package can be backup and restore without error
|
||||
# # This is from the linter, tests if app is the Yunohost-apps organization
|
||||
# [ $RESULT_linter_level_6 -eq 1 ] || \
|
||||
# ([ $RESULT_linter_level_6 -eq 0 ] && \
|
||||
# [ "${level[6]}" == "2" ] )
|
||||
# }
|
||||
#
|
||||
# pass_level_7() {
|
||||
# # -> None errors in all tests performed
|
||||
# # Validated if none errors is happened.
|
||||
# [ $RESULT_check_subdir -ne -1 ] && \
|
||||
# [ $RESULT_check_upgrade -ne -1 ] && \
|
||||
# [ $RESULT_check_private -ne -1 ] && \
|
||||
# [ $RESULT_check_multi_instance -ne -1 ] && \
|
||||
# [ $RESULT_check_port -ne -1 ] && \
|
||||
# [ $RESULT_check_backup -ne -1 ] && \
|
||||
# [ $RESULT_check_restore -ne -1 ] && \
|
||||
# [ $RESULT_change_url -ne -1 ] && \
|
||||
# [ $RESULT_action_config_panel -ne -1 ] && \
|
||||
# ([ $RESULT_linter_level_7 -ge 1 ] ||
|
||||
# ([ $RESULT_linter_level_7 -eq 0 ] && \
|
||||
# [ "${level[8]}" == "2" ] ))
|
||||
# }
|
||||
#
|
||||
# pass_level_8() {
|
||||
# # This happens in the linter
|
||||
# # When writing this, defined as app being maintained + long term quality (=
|
||||
# # certain amount of time level 5+ in the last year)
|
||||
# [ $RESULT_linter_level_8 -ge 1 ] || \
|
||||
# ([ $RESULT_linter_level_8 -eq 0 ] && \
|
||||
# [ "${level[8]}" == "2" ] )
|
||||
# }
|
||||
#
|
||||
# pass_level_9() {
|
||||
# list_url="https://raw.githubusercontent.com/YunoHost/apps/master/apps.json"
|
||||
# curl --silent $list_url | jq ".[\"$app_id\"].high_quality" | grep -q "true"
|
||||
# }
|
||||
#
|
||||
# # Check if the level can be changed
|
||||
# level_can_change () {
|
||||
# # If the level is set at auto, it's waiting for a change
|
||||
# # And if it's set at 2, its value can be modified by a new result
|
||||
# [ "${level[$1]}" == "auto" ] || [ "${level[$1]}" -eq 2 ]
|
||||
# }
|
||||
#
|
||||
# if level_can_change 1; then pass_level_1 && level[1]=2 || level[1]=0; fi
|
||||
# if level_can_change 2; then pass_level_2 && level[2]=2 || level[2]=0; fi
|
||||
# if level_can_change 3; then pass_level_3 && level[3]=2 || level[3]=0; fi
|
||||
# if level_can_change 4; then pass_level_4 && level[4]=2 || level[4]=0; fi
|
||||
# if level_can_change 5; then pass_level_5 && level[5]=2 || level[5]=0; fi
|
||||
# if level_can_change 6; then pass_level_6 && level[6]=2 || level[6]=0; fi
|
||||
# if level_can_change 7; then pass_level_7 && level[7]=2 || level[7]=0; fi
|
||||
# if level_can_change 8; then pass_level_8 && level[8]=2 || level[8]=0; fi
|
||||
# if level_can_change 9; then pass_level_9 && level[9]=2 || level[9]=0; fi
|
||||
#
|
||||
# # Level 10 has no definition yet
|
||||
# level[10]=0
|
||||
#
|
||||
# # Initialize the global level
|
||||
# global_level=0
|
||||
#
|
||||
# # Calculate the final level
|
||||
# for i in `seq 1 10`
|
||||
# do
|
||||
#
|
||||
# # If there is a level still at 'auto', it's a mistake.
|
||||
# if [ "${level[i]}" == "auto" ]
|
||||
# then
|
||||
# # So this level will set at 0.
|
||||
# level[i]=0
|
||||
#
|
||||
# # If the level is at 1 or 2. The global level will be set at this level
|
||||
# elif [ "${level[i]}" -ge 1 ]
|
||||
# then
|
||||
# global_level=$i
|
||||
#
|
||||
# # But, if the level is at 0, the loop stop here
|
||||
# # Like that, the global level rise while none level have failed
|
||||
# else
|
||||
# break
|
||||
# fi
|
||||
# done
|
||||
#
|
||||
# # If some witness files was missing, it's a big error ! So, the level fall immediately at 0.
|
||||
# if [ $RESULT_witness -eq 1 ]
|
||||
# then
|
||||
# log_error "Some witness files has been deleted during those tests ! It's a very bad thing !"
|
||||
# global_level=0
|
||||
# fi
|
||||
#
|
||||
# # If the package linter returned a critical error, the app is flagged as broken / level 0
|
||||
# if [ $RESULT_linter_broken -eq 1 ]
|
||||
# then
|
||||
# log_error "The package linter reported a critical failure ! App is considered broken !"
|
||||
# global_level=0
|
||||
# fi
|
||||
#
|
||||
# if [ $RESULT_alias_traversal -eq 1 ]
|
||||
# then
|
||||
# log_error "Issue alias_traversal was detected ! Please see here https://github.com/YunoHost/example_ynh/pull/45 to fix that."
|
||||
# fi
|
||||
#
|
||||
# # Then, print the levels
|
||||
# # Print the global level
|
||||
# verbose_level=$(grep "^$global_level " "./levels.list" | cut -c4-)
|
||||
#
|
||||
# log_info "Level of this application: $global_level ($verbose_level)"
|
||||
#
|
||||
# # And print the value for each level
|
||||
# for i in `seq 1 10`
|
||||
# do
|
||||
# display="0"
|
||||
# if [ "${level[$i]}" -ge 1 ]; then
|
||||
# display="1"
|
||||
# fi
|
||||
# echo -e "\t Level $i: $display"
|
||||
# done
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ clean_exit () {
|
|||
|
||||
LXC_RESET
|
||||
|
||||
[ -n "$TEST_CONTEXT" ] rm -rf "$TEST_CONTEXT"
|
||||
[ -n "$TEST_CONTEXT" ] && rm -rf "$TEST_CONTEXT"
|
||||
rm -f "$lock_file"
|
||||
|
||||
exit $1
|
||||
|
|
@ -78,7 +78,7 @@ readonly WHITE=$(printf '\033[39m')
|
|||
|
||||
function log_title()
|
||||
{
|
||||
cat << EOF
|
||||
cat << EOF | tee -a /proc/self/fd/3
|
||||
${BOLD}
|
||||
===================================
|
||||
$1
|
||||
|
|
@ -89,51 +89,51 @@ EOF
|
|||
|
||||
function log_small_title()
|
||||
{
|
||||
echo -e "\n${BOLD} > ${1}${NORMAL}\n"
|
||||
echo -e "\n${BOLD} > ${1}${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
|
||||
function log_debug()
|
||||
{
|
||||
echo "$1" >&3
|
||||
echo "$1" >> /proc/self/fd/3
|
||||
}
|
||||
|
||||
function log_info()
|
||||
{
|
||||
echo "${1}"
|
||||
echo "${1}" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
function log_success()
|
||||
{
|
||||
echo "${BOLD}${GREEN}Success: ${1}${NORMAL}"
|
||||
echo "${BOLD}${GREEN}Success: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
function log_warning()
|
||||
{
|
||||
echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}"
|
||||
echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
function log_error()
|
||||
{
|
||||
echo "${BOLD}${RED}Error: ${1}${NORMAL}"
|
||||
echo "${BOLD}${RED}Error: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
function log_critical()
|
||||
{
|
||||
echo "${BOLD}${RED}Critical: ${1}${NORMAL}"
|
||||
echo "${BOLD}${RED}Critical: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||
clean_exit 1
|
||||
}
|
||||
|
||||
function log_report_test_success () {
|
||||
echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n"
|
||||
echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
function log_report_test_warning () {
|
||||
echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n"
|
||||
echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
function log_report_test_failed () {
|
||||
echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n"
|
||||
echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||
}
|
||||
|
||||
#=================================================
|
||||
|
|
|
|||
70
lib/tests.sh
70
lib/tests.sh
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
#=================================================
|
||||
# Logistic helpers
|
||||
# "Low-level" logistic helpers
|
||||
#=================================================
|
||||
|
||||
_RUN_YUNOHOST_CMD() {
|
||||
|
|
@ -258,19 +258,24 @@ PACKAGE_LINTER () {
|
|||
# Execute package linter and linter_result gets the return code of the package linter
|
||||
./package_linter/package_linter.py "$package_path" | tee -a "$complete_log"
|
||||
./package_linter/package_linter.py "$package_path" --json | tee -a "$complete_log" > $current_test_results
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
TEST_INSTALL () {
|
||||
# Try to install in a sub path, on root or without url access
|
||||
# $1 = install type
|
||||
|
||||
local install_type=$1
|
||||
|
||||
# This is a separate case ... at least from an hystorical point of view ...
|
||||
# but it helpers for semantic that the test is a "TEST_INSTALL" ...
|
||||
[ "$install_type" = "multi" ] && { _TEST_MULTI_INSTANCE; return $?; }
|
||||
|
||||
local check_path="/"
|
||||
local is_public="1"
|
||||
[ "$install_type" = "subdir" ] && { start_test "Installation in a sub path"; local check_path=/path; }
|
||||
[ "$install_type" = "root" ] && { start_test "Installation on the root"; }
|
||||
[ "$install_type" = "nourl" ] && { start_test "Installation without url access"; local check_path=""; }
|
||||
[ "$install_type" = "private" ] && { start_test "Installation in private mode"; local is_public="0"; }
|
||||
[ "$install_type" = "subdir" ] && { start_test "Installation in a sub path"; local check_path=/path; }
|
||||
[ "$install_type" = "root" ] && { start_test "Installation on the root"; }
|
||||
[ "$install_type" = "nourl" ] && { start_test "Installation without url access"; local check_path=""; }
|
||||
[ "$install_type" = "private" ] && { start_test "Installation in private mode"; local is_public="0"; }
|
||||
local snapname=snap_${install_type}install
|
||||
|
||||
LOAD_LXC_SNAPSHOT snap0
|
||||
|
|
@ -298,6 +303,29 @@ TEST_INSTALL () {
|
|||
return $?
|
||||
}
|
||||
|
||||
_TEST_MULTI_INSTANCE () {
|
||||
|
||||
start_test "Multi-instance installations"
|
||||
|
||||
# Check if an install have previously work
|
||||
at_least_one_install_succeeded || return 1
|
||||
|
||||
local check_path=$(default_install_path)
|
||||
|
||||
LOAD_LXC_SNAPSHOT snap0
|
||||
|
||||
log_small_title "First installation: path=$DOMAIN$check_path" \
|
||||
&&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \
|
||||
&& log_small_title "Second installation: path=$SUBDOMAIN$check_path" \
|
||||
&& _INSTALL_APP "path=$check_path" \
|
||||
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \
|
||||
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2 \
|
||||
&& _REMOVE_APP \
|
||||
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
TEST_UPGRADE () {
|
||||
|
||||
local commit=$1
|
||||
|
|
@ -351,27 +379,6 @@ TEST_UPGRADE () {
|
|||
return $?
|
||||
}
|
||||
|
||||
TEST_MULTI_INSTANCE () {
|
||||
|
||||
start_test "Multi-instance installations"
|
||||
|
||||
# Check if an install have previously work
|
||||
at_least_one_install_succeeded || return 1
|
||||
|
||||
local check_path=$(default_install_path)
|
||||
|
||||
LOAD_LXC_SNAPSHOT snap0
|
||||
|
||||
log_small_title "First installation: path=$DOMAIN$check_path" \
|
||||
&&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \
|
||||
&& log_small_title "Second installation: path=$SUBDOMAIN$check_path" \
|
||||
&&_INSTALL_APP "path=$check_path" \
|
||||
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \
|
||||
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
TEST_PORT_ALREADY_USED () {
|
||||
|
||||
start_test "Port already used"
|
||||
|
|
@ -464,6 +471,7 @@ TEST_BACKUP_RESTORE () {
|
|||
|
||||
# Place the copy of the backup archive in the container.
|
||||
sudo lxc file push -r ./ynh_backups $LXC_NAME/home/yunohost.backup/archives/
|
||||
RUN_INSIDE_LXC ls -l /home/yunohost.backup/archives/
|
||||
|
||||
log_small_title "Restore on a clean YunoHost system..."
|
||||
fi
|
||||
|
|
@ -489,10 +497,10 @@ TEST_CHANGE_URL () {
|
|||
# Check if an install have previously work
|
||||
at_least_one_install_succeeded || return 1
|
||||
this_is_a_web_app || return 0
|
||||
|
||||
|
||||
log_small_title "Preliminary install..." \
|
||||
&& _LOAD_SNAPSHOT_OR_INSTALL_APP "/"
|
||||
|
||||
|
||||
local ret=$?
|
||||
[ $ret -eq 0 ] || { return 1; }
|
||||
|
||||
|
|
@ -548,7 +556,7 @@ TEST_CHANGE_URL () {
|
|||
|
||||
|
||||
ACTIONS_CONFIG_PANEL () {
|
||||
|
||||
|
||||
test_type=$1
|
||||
|
||||
# Define a function to split a file in multiple parts. Used for actions and config-panel toml
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ source lib/witness.sh
|
|||
complete_log="./Complete.log"
|
||||
|
||||
# Purge some log files
|
||||
> "$complete_log"
|
||||
rm -f "$complete_log" && touch "$complete_log"
|
||||
|
||||
# Redirect fd 3 (=debug steam) to complete log
|
||||
exec 3>>$complete_log
|
||||
|
|
@ -16,9 +16,6 @@ exec 3>>$complete_log
|
|||
# Parse the check_process and generate jsons that describe tests to run
|
||||
#=======================================================================
|
||||
|
||||
# Parse the check_process only if it's exist
|
||||
check_process="$package_path/check_process"
|
||||
|
||||
# Extract a section found between $1 and $2 from the file $3
|
||||
extract_check_process_section () {
|
||||
local source_file="${3:-$check_process}"
|
||||
|
|
@ -156,11 +153,11 @@ parse_check_process() {
|
|||
fi
|
||||
|
||||
is_test_enabled pkg_linter && add_test "PACKAGE_LINTER"
|
||||
is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir"
|
||||
is_test_enabled setup_root && add_test "TEST_INSTALL" "root"
|
||||
is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir"
|
||||
is_test_enabled setup_nourl && add_test "TEST_INSTALL" "nourl"
|
||||
is_test_enabled setup_private && add_test "TEST_INSTALL" "private"
|
||||
is_test_enabled multi_instance && add_test "TEST_MULTI_INSTANCE"
|
||||
is_test_enabled multi_instance && add_test "TEST_INSTALL" "multi"
|
||||
is_test_enabled backup_restore && add_test "TEST_BACKUP_RESTORE"
|
||||
|
||||
# Upgrades
|
||||
|
|
@ -220,15 +217,15 @@ guess_test_configuration() {
|
|||
local install_args=$(python "./lib/manifest_parsing.py" "$package_path/manifest.json" | cut -d ':' -f1,2 | tr ':' '=' | tr '\n' '&')
|
||||
|
||||
add_test "PACKAGE_LINTER"
|
||||
add_test "TEST_INSTALL subdir"
|
||||
add_test "TEST_INSTALL root"
|
||||
add_test "TEST_INSTALL" "root"
|
||||
add_test "TEST_INSTALL" "subdir"
|
||||
if echo $install_args | grep -q "is_public="
|
||||
then
|
||||
add_test "TEST_INSTALL" "private"
|
||||
fi
|
||||
if grep multi_instance "$package_path/manifest.json" | grep -q true
|
||||
then
|
||||
add_test "TEST_MULTI_INSTANCE"
|
||||
add_test "TEST_INSTALL" "multi"
|
||||
fi
|
||||
add_test "TEST_BACKUP_RESTORE"
|
||||
add_test "TEST_UPGRADE"
|
||||
|
|
@ -243,6 +240,9 @@ run_all_tests() {
|
|||
mkdir -p $TEST_CONTEXT/tests
|
||||
mkdir -p $TEST_CONTEXT/results
|
||||
|
||||
# Parse the check_process only if it's exist
|
||||
check_process="$package_path/check_process"
|
||||
|
||||
[ -e "$check_process" ] \
|
||||
&& parse_check_process \
|
||||
|| guess_test_configuration
|
||||
|
|
@ -281,7 +281,7 @@ run_all_tests() {
|
|||
done
|
||||
|
||||
# Print the final results of the tests
|
||||
# FIXME COMPUTE_RESULTS_SUMMARY $test_serie_id
|
||||
python3 lib/analyze_test_results.py $TEST_CONTEXT 2>$TEST_CONTEXT/summary.json
|
||||
|
||||
# Restore the started time for the timer
|
||||
starttime=$complete_start_timer
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue