mirror of
https://github.com/YunoHost/package_check.git
synced 2024-09-03 20:06:20 +02:00
Moar fixes + reimplement summary/level analysis
This commit is contained in:
Alexandre Aubin
parent
0415e0fcf5
commit
4937a503ea
4 changed files with 276 additions and 329 deletions
|
|
@ -1,284 +1,223 @@
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
# Levels
|
def load_tests(test_folder):
|
||||||
|
|
||||||
#0 Broken
|
for test in sorted(os.listdir(test_folder + "/tests")):
|
||||||
#1 Installable
|
|
||||||
#2 Installable in all situations
|
|
||||||
#3 Can be updated
|
|
||||||
#4 Backup and restore support
|
|
||||||
#5 Clean
|
|
||||||
#6 Open to contributions from the community
|
|
||||||
#7 Successfully pass all functional tests and linter tests
|
|
||||||
#8 Maintained and long-term good quality
|
|
||||||
#9 High quality app
|
|
||||||
#10 Package assessed as perfect
|
|
||||||
|
|
||||||
# Linter stuff:
|
j = json.load(open(test_folder + "/tests/" + test))
|
||||||
|
j["id"] = os.path.basename(test).split(".")[0]
|
||||||
|
j["results"] = json.load(open(test_folder + "/results/" + j["id"] + ".json"))
|
||||||
|
yield j
|
||||||
|
|
||||||
|
|
||||||
# # Check we qualify for level 6, 7, 8
|
levels = []
|
||||||
# # Linter will have a warning called "app_in_github_org" if app ain't in the
|
|
||||||
# # yunohost-apps org...
|
|
||||||
# if ! cat "./temp_linter_result.json" | jq ".warning" | grep -q "app_in_github_org"
|
def level(level_, descr):
|
||||||
# then
|
def decorator(f):
|
||||||
# local pass_level_6="true"
|
f.descr = descr
|
||||||
# fi
|
f.level = level_
|
||||||
# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_7"
|
levels.insert(level_, f)
|
||||||
# then
|
return f
|
||||||
# local pass_level_7="true"
|
return decorator
|
||||||
# fi
|
|
||||||
# if cat "./temp_linter_result.json" | jq ".success" | grep -q "qualify_for_level_8"
|
|
||||||
# then
|
###############################################################################
|
||||||
# local pass_level_8="true"
|
|
||||||
# fi
|
|
||||||
#
|
@level(0, "Broken")
|
||||||
# # If there are any critical errors, we'll force level 0
|
def level_0(tests):
|
||||||
# if [[ -n "$(cat "./temp_linter_result.json" | jq ".critical" | grep -v '\[\]')" ]]
|
return True
|
||||||
# then
|
|
||||||
# local pass_level_0="false"
|
|
||||||
# # If there are any regular errors, we'll cap to 4
|
@level(1, "Installable in at least one scenario")
|
||||||
# elif [[ -n "$(cat "./temp_linter_result.json" | jq ".error" | grep -v '\[\]')" ]]
|
def level_1(tests):
|
||||||
# then
|
"""
|
||||||
# local pass_level_4="false"
|
Any install test succeded
|
||||||
# # Otherwise, test pass (we'll display a warning depending on if there are
|
And there are no critical issues in the linter
|
||||||
# # any remaning warnings or not)
|
"""
|
||||||
# else
|
|
||||||
# if [[ -n "$(cat "./temp_linter_result.json" | jq ".warning" | grep -v '\[\]')" ]]
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||||
# then
|
install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
|
||||||
# log_report_test_warning
|
witness_missing_detected = any(t["results"].get("witness") for t in tests)
|
||||||
# else
|
|
||||||
# log_report_test_success
|
return linter_tests != [] \
|
||||||
# fi
|
and linter_tests[0]["results"]["critical"] == [] \
|
||||||
# local pass_level_4="true"
|
and not witness_missing_detected \
|
||||||
# fi
|
and any(t["results"]["main_result"] == "success" for t in install_tests)
|
||||||
|
|
||||||
|
|
||||||
|
@level(2, "Installable in all scenarios")
|
||||||
|
def level_2(tests):
|
||||||
|
"""
|
||||||
|
All install tests succeeded (and at least one test was made)
|
||||||
|
"""
|
||||||
|
|
||||||
|
install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
|
||||||
|
|
||||||
|
return install_tests != [] \
|
||||||
|
and all(t["results"]["main_result"] == "success" for t in install_tests)
|
||||||
|
|
||||||
|
|
||||||
|
@level(3, "Can be upgraded")
|
||||||
|
def level_3(tests):
|
||||||
|
"""
|
||||||
|
All upgrade tests succeeded (and at least one test was made)
|
||||||
|
"""
|
||||||
|
|
||||||
|
upgrade_tests = [t for t in tests if t["test_type"] == "TEST_UPGRADE"]
|
||||||
|
|
||||||
|
return upgrade_tests != [] \
|
||||||
|
and all(t["results"]["main_result"] == "success" for t in upgrade_tests)
|
||||||
|
|
||||||
|
|
||||||
|
@level(4, "Can be backup/restored")
|
||||||
|
def level_4(tests):
|
||||||
|
"""
|
||||||
|
All backup/restore tests succeded (and at least one test was made)
|
||||||
|
"""
|
||||||
|
|
||||||
|
backup_tests = [t for t in tests if t["test_type"] == "TEST_BACKUP"]
|
||||||
|
|
||||||
|
return backup_tests != [] \
|
||||||
|
and all(t["results"]["main_result"] == "success" for t in backup_tests)
|
||||||
|
|
||||||
|
|
||||||
|
@level(5, "No linter errors")
|
||||||
|
def level_5(tests):
|
||||||
|
"""
|
||||||
|
Linter returned no errors (= main_result is success)
|
||||||
|
and no alias/path traversal issue detected during tests
|
||||||
|
"""
|
||||||
|
|
||||||
|
alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests)
|
||||||
|
|
||||||
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||||
|
|
||||||
|
return not alias_traversal_detected \
|
||||||
|
and linter_tests != [] \
|
||||||
|
and linter_tests[0]["results"]["main_result"] == "success"
|
||||||
|
|
||||||
|
|
||||||
|
@level(6, "App is in a community-operated git org")
|
||||||
|
def level_6(tests):
|
||||||
|
"""
|
||||||
|
The app is in the Yunohost-Apps organization
|
||||||
|
(the linter will report a warning named "is_in_github_org" if it's not)
|
||||||
|
"""
|
||||||
|
|
||||||
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||||
|
|
||||||
|
return linter_tests != [] \
|
||||||
|
and "is_in_github_org" not in linter_tests[0]["results"]["warning"]
|
||||||
|
|
||||||
|
|
||||||
|
@level(7, "Pass all tests + no linter warnings")
|
||||||
|
def level_7(tests):
|
||||||
|
"""
|
||||||
|
All tests succeeded + no warning in linter (that part is tested by the
|
||||||
|
# linter which will report a "qualify_for_level_7" in successes)
|
||||||
|
"""
|
||||||
|
|
||||||
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||||
|
|
||||||
|
return all(t["results"]["main_result"] == "success" for t in tests) \
|
||||||
|
and linter_tests != [] \
|
||||||
|
and "qualify_for_level_7" in linter_tests[0]["results"]["success"]
|
||||||
|
|
||||||
|
|
||||||
|
@level(8, "Maintained and long-term good quality")
|
||||||
|
def level_8(tests):
|
||||||
|
"""
|
||||||
|
App is maintained and long-term good quality (this is tested by the linter
|
||||||
|
which will report a "qualify_for_level_8")
|
||||||
|
"""
|
||||||
|
|
||||||
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||||
|
|
||||||
|
return linter_tests != [] \
|
||||||
|
and "qualify_for_level_8" in linter_tests[0]["results"]["success"]
|
||||||
|
|
||||||
|
|
||||||
|
@level(9, "Flagged high-quality in app catalog")
|
||||||
|
def level_9(tests):
|
||||||
|
"""
|
||||||
|
App is flagged high-quality in the app catalog (this is tested by the linter
|
||||||
|
which will rpeort a "qualify_for_level_9")
|
||||||
|
"""
|
||||||
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
||||||
|
|
||||||
|
return linter_tests != [] \
|
||||||
|
and "qualify_for_level_9" in linter_tests[0]["results"]["success"]
|
||||||
|
|
||||||
|
|
||||||
|
tests = list(load_tests(sys.argv[1]))
|
||||||
|
|
||||||
|
test_types = {
|
||||||
|
"PACKAGE_LINTER": "Package linter",
|
||||||
|
"TEST_INSTALL": "Install",
|
||||||
|
"TEST_UPGRADE": "Upgrade",
|
||||||
|
"TEST_BACKUP_RESTORE": "Backup/restore",
|
||||||
|
"TEST_CHANGE_URL": "Change url",
|
||||||
|
"TEST_PORT_ALREADY_USED": "Port already used",
|
||||||
|
"ACTIONS_CONFIG_PANEL": "Config/panel"
|
||||||
|
}
|
||||||
|
|
||||||
|
OK = ' \033[1m\033[92mOK\033[0m '
|
||||||
|
FAIL = '\033[91mfail\033[0m'
|
||||||
|
|
||||||
|
print("Tests summary")
|
||||||
|
print("=============")
|
||||||
|
for test in tests:
|
||||||
|
test_display_name = test_types[test["test_type"]]
|
||||||
|
if test["test_arg"]:
|
||||||
|
test_display_name += " (%s)" % test["test_arg"][:8]
|
||||||
|
test_display_name += ":"
|
||||||
|
|
||||||
|
result = OK if test["results"]["main_result"] == "success" else FAIL
|
||||||
|
print("{test: <25}{result}".format(test=test_display_name, result=result))
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("Level results")
|
||||||
|
print("=============")
|
||||||
|
|
||||||
|
stop_global_level_bump = False
|
||||||
|
|
||||||
|
global_level = level_0
|
||||||
|
|
||||||
|
for level in levels[1:]:
|
||||||
|
level.passed = level(tests)
|
||||||
|
|
||||||
|
if not level.passed:
|
||||||
|
stop_global_level_bump = True
|
||||||
|
|
||||||
|
if not stop_global_level_bump:
|
||||||
|
global_level = level
|
||||||
|
display = OK
|
||||||
|
else:
|
||||||
|
display = " ok " if level.passed else ""
|
||||||
|
|
||||||
|
print("Level {i} {descr: <40} {result}".format(i=level.level,
|
||||||
|
descr="(%s)"%level.descr[:38], result=display))
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("\033[1mGlobal level for this application: %s (%s)\033[0m" % (global_level.level, global_level.descr))
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"tests": [{
|
||||||
|
"test_type": t["test_type"],
|
||||||
|
"test_arg": t["test_arg"],
|
||||||
|
"main_result": t["results"]["main_result"]
|
||||||
|
} for t in tests],
|
||||||
|
"levels": {level.level: level.passed for level in levels[1:]},
|
||||||
|
"global_level": global_level.level
|
||||||
|
}
|
||||||
|
|
||||||
|
sys.stderr.write(json.dumps(summary))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#
|
|
||||||
# local test_serie_id=$1
|
|
||||||
# source $TEST_CONTEXT/$test_serie_id/results
|
|
||||||
#
|
|
||||||
# # Print the test result
|
|
||||||
# print_result () {
|
|
||||||
#
|
|
||||||
# # Print the result of this test
|
|
||||||
# # we use printf to force the length to 30 (filled with space)
|
|
||||||
# testname=$(printf %-30.30s "$1:")
|
|
||||||
# if [ $2 -eq 1 ]
|
|
||||||
# then
|
|
||||||
# echo "$testname ${BOLD}${GREEN}SUCCESS${NORMAL}"
|
|
||||||
# elif [ $2 -eq -1 ]
|
|
||||||
# then
|
|
||||||
# echo "$testname ${BOLD}${RED}FAIL${NORMAL}"
|
|
||||||
# else
|
|
||||||
# echo "$testname Not evaluated."
|
|
||||||
# fi
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# # Print the result for each test
|
|
||||||
# echo -e "\n\n"
|
|
||||||
# print_result "Package linter" $RESULT_linter
|
|
||||||
# print_result "Install (root)" $RESULT_check_root
|
|
||||||
# print_result "Install (subpath)" $RESULT_check_subdir
|
|
||||||
# print_result "Install (no url)" $RESULT_check_nourl
|
|
||||||
# print_result "Install (private)" $RESULT_check_private
|
|
||||||
# print_result "Install (multi-instance)" $RESULT_check_multi_instance
|
|
||||||
# print_result "Upgrade" $RESULT_check_upgrade
|
|
||||||
# print_result "Backup" $RESULT_check_backup
|
|
||||||
# print_result "Restore" $RESULT_check_restore
|
|
||||||
# print_result "Change URL" $RESULT_change_url
|
|
||||||
# print_result "Port already used" $RESULT_check_port
|
|
||||||
# print_result "Actions and config-panel" $RESULT_action_config_panel
|
|
||||||
#
|
|
||||||
# # Determine the level for this app
|
|
||||||
#
|
|
||||||
# # Each level can has 5 different values
|
|
||||||
# # 0 -> If this level can't be validated
|
|
||||||
# # 1 -> If this level is forced. Even if the tests fails
|
|
||||||
# # 2 -> Indicates the tests had previously validated this level
|
|
||||||
# # auto -> This level has not a value yet.
|
|
||||||
# # na -> This level will not be checked, but it'll be ignored in the final sum
|
|
||||||
#
|
|
||||||
# # Set default values for level, if they're empty.
|
|
||||||
# test -n "${level[1]}" || level[1]=auto
|
|
||||||
# test -n "${level[2]}" || level[2]=auto
|
|
||||||
# test -n "${level[3]}" || level[3]=auto
|
|
||||||
# test -n "${level[4]}" || level[4]=auto
|
|
||||||
# test -n "${level[5]}" || level[5]=auto
|
|
||||||
# test -n "${level[5]}" || level[5]=auto
|
|
||||||
# test -n "${level[6]}" || level[6]=auto
|
|
||||||
# test -n "${level[7]}" || level[7]=auto
|
|
||||||
# test -n "${level[8]}" || level[8]=auto
|
|
||||||
# test -n "${level[9]}" || level[9]=0
|
|
||||||
# test -n "${level[10]}" || level[10]=0
|
|
||||||
#
|
|
||||||
# pass_level_1() {
|
|
||||||
# # FIXME FIXME #FIXME
|
|
||||||
# return 0
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_2() {
|
|
||||||
# # -> The package can be install and remove in all tested configurations.
|
|
||||||
# # Validated if none install failed
|
|
||||||
# [ $RESULT_check_subdir -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_root -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_private -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_multi_instance -ne -1 ]
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_3() {
|
|
||||||
# # -> The package can be upgraded from the same version.
|
|
||||||
# # Validated if the upgrade is ok. Or if the upgrade has been not tested but already validated before.
|
|
||||||
# [ $RESULT_check_upgrade -eq 1 ] || \
|
|
||||||
# ( [ $RESULT_check_upgrade -ne -1 ] && \
|
|
||||||
# [ "${level[3]}" == "2" ] )
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_4() {
|
|
||||||
# # -> The package can be backup and restore without error
|
|
||||||
# # Validated if backup and restore are ok. Or if backup and restore have been not tested but already validated before.
|
|
||||||
# ( [ $RESULT_check_backup -eq 1 ] && \
|
|
||||||
# [ $RESULT_check_restore -eq 1 ] ) || \
|
|
||||||
# ( [ $RESULT_check_backup -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_restore -ne -1 ] && \
|
|
||||||
# [ "${level[4]}" == "2" ] )
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_5() {
|
|
||||||
# # -> The package have no error with package linter
|
|
||||||
# # -> The package does not have any alias_traversal error
|
|
||||||
# # Validated if Linter is ok. Or if Linter has been not tested but already validated before.
|
|
||||||
# [ $RESULT_alias_traversal -ne 1 ] && \
|
|
||||||
# ([ $RESULT_linter -ge 1 ] || \
|
|
||||||
# ( [ $RESULT_linter -eq 0 ] && \
|
|
||||||
# [ "${level[5]}" == "2" ] ) )
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_6() {
|
|
||||||
# # -> The package can be backup and restore without error
|
|
||||||
# # This is from the linter, tests if app is the Yunohost-apps organization
|
|
||||||
# [ $RESULT_linter_level_6 -eq 1 ] || \
|
|
||||||
# ([ $RESULT_linter_level_6 -eq 0 ] && \
|
|
||||||
# [ "${level[6]}" == "2" ] )
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_7() {
|
|
||||||
# # -> None errors in all tests performed
|
|
||||||
# # Validated if none errors is happened.
|
|
||||||
# [ $RESULT_check_subdir -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_upgrade -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_private -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_multi_instance -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_port -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_backup -ne -1 ] && \
|
|
||||||
# [ $RESULT_check_restore -ne -1 ] && \
|
|
||||||
# [ $RESULT_change_url -ne -1 ] && \
|
|
||||||
# [ $RESULT_action_config_panel -ne -1 ] && \
|
|
||||||
# ([ $RESULT_linter_level_7 -ge 1 ] ||
|
|
||||||
# ([ $RESULT_linter_level_7 -eq 0 ] && \
|
|
||||||
# [ "${level[8]}" == "2" ] ))
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_8() {
|
|
||||||
# # This happens in the linter
|
|
||||||
# # When writing this, defined as app being maintained + long term quality (=
|
|
||||||
# # certain amount of time level 5+ in the last year)
|
|
||||||
# [ $RESULT_linter_level_8 -ge 1 ] || \
|
|
||||||
# ([ $RESULT_linter_level_8 -eq 0 ] && \
|
|
||||||
# [ "${level[8]}" == "2" ] )
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# pass_level_9() {
|
|
||||||
# list_url="https://raw.githubusercontent.com/YunoHost/apps/master/apps.json"
|
|
||||||
# curl --silent $list_url | jq ".[\"$app_id\"].high_quality" | grep -q "true"
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# # Check if the level can be changed
|
|
||||||
# level_can_change () {
|
|
||||||
# # If the level is set at auto, it's waiting for a change
|
|
||||||
# # And if it's set at 2, its value can be modified by a new result
|
|
||||||
# [ "${level[$1]}" == "auto" ] || [ "${level[$1]}" -eq 2 ]
|
|
||||||
# }
|
|
||||||
#
|
|
||||||
# if level_can_change 1; then pass_level_1 && level[1]=2 || level[1]=0; fi
|
|
||||||
# if level_can_change 2; then pass_level_2 && level[2]=2 || level[2]=0; fi
|
|
||||||
# if level_can_change 3; then pass_level_3 && level[3]=2 || level[3]=0; fi
|
|
||||||
# if level_can_change 4; then pass_level_4 && level[4]=2 || level[4]=0; fi
|
|
||||||
# if level_can_change 5; then pass_level_5 && level[5]=2 || level[5]=0; fi
|
|
||||||
# if level_can_change 6; then pass_level_6 && level[6]=2 || level[6]=0; fi
|
|
||||||
# if level_can_change 7; then pass_level_7 && level[7]=2 || level[7]=0; fi
|
|
||||||
# if level_can_change 8; then pass_level_8 && level[8]=2 || level[8]=0; fi
|
|
||||||
# if level_can_change 9; then pass_level_9 && level[9]=2 || level[9]=0; fi
|
|
||||||
#
|
|
||||||
# # Level 10 has no definition yet
|
|
||||||
# level[10]=0
|
|
||||||
#
|
|
||||||
# # Initialize the global level
|
|
||||||
# global_level=0
|
|
||||||
#
|
|
||||||
# # Calculate the final level
|
|
||||||
# for i in `seq 1 10`
|
|
||||||
# do
|
|
||||||
#
|
|
||||||
# # If there is a level still at 'auto', it's a mistake.
|
|
||||||
# if [ "${level[i]}" == "auto" ]
|
|
||||||
# then
|
|
||||||
# # So this level will set at 0.
|
|
||||||
# level[i]=0
|
|
||||||
#
|
|
||||||
# # If the level is at 1 or 2. The global level will be set at this level
|
|
||||||
# elif [ "${level[i]}" -ge 1 ]
|
|
||||||
# then
|
|
||||||
# global_level=$i
|
|
||||||
#
|
|
||||||
# # But, if the level is at 0, the loop stop here
|
|
||||||
# # Like that, the global level rise while none level have failed
|
|
||||||
# else
|
|
||||||
# break
|
|
||||||
# fi
|
|
||||||
# done
|
|
||||||
#
|
|
||||||
# # If some witness files was missing, it's a big error ! So, the level fall immediately at 0.
|
|
||||||
# if [ $RESULT_witness -eq 1 ]
|
|
||||||
# then
|
|
||||||
# log_error "Some witness files has been deleted during those tests ! It's a very bad thing !"
|
|
||||||
# global_level=0
|
|
||||||
# fi
|
|
||||||
#
|
|
||||||
# # If the package linter returned a critical error, the app is flagged as broken / level 0
|
|
||||||
# if [ $RESULT_linter_broken -eq 1 ]
|
|
||||||
# then
|
|
||||||
# log_error "The package linter reported a critical failure ! App is considered broken !"
|
|
||||||
# global_level=0
|
|
||||||
# fi
|
|
||||||
#
|
|
||||||
# if [ $RESULT_alias_traversal -eq 1 ]
|
|
||||||
# then
|
|
||||||
# log_error "Issue alias_traversal was detected ! Please see here https://github.com/YunoHost/example_ynh/pull/45 to fix that."
|
|
||||||
# fi
|
|
||||||
#
|
|
||||||
# # Then, print the levels
|
|
||||||
# # Print the global level
|
|
||||||
# verbose_level=$(grep "^$global_level " "./levels.list" | cut -c4-)
|
|
||||||
#
|
|
||||||
# log_info "Level of this application: $global_level ($verbose_level)"
|
|
||||||
#
|
|
||||||
# # And print the value for each level
|
|
||||||
# for i in `seq 1 10`
|
|
||||||
# do
|
|
||||||
# display="0"
|
|
||||||
# if [ "${level[$i]}" -ge 1 ]; then
|
|
||||||
# display="1"
|
|
||||||
# fi
|
|
||||||
# echo -e "\t Level $i: $display"
|
|
||||||
# done
|
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ clean_exit () {
|
||||||
|
|
||||||
LXC_RESET
|
LXC_RESET
|
||||||
|
|
||||||
[ -n "$TEST_CONTEXT" ] rm -rf "$TEST_CONTEXT"
|
[ -n "$TEST_CONTEXT" ] && rm -rf "$TEST_CONTEXT"
|
||||||
rm -f "$lock_file"
|
rm -f "$lock_file"
|
||||||
|
|
||||||
exit $1
|
exit $1
|
||||||
|
|
@ -78,7 +78,7 @@ readonly WHITE=$(printf '\033[39m')
|
||||||
|
|
||||||
function log_title()
|
function log_title()
|
||||||
{
|
{
|
||||||
cat << EOF
|
cat << EOF | tee -a /proc/self/fd/3
|
||||||
${BOLD}
|
${BOLD}
|
||||||
===================================
|
===================================
|
||||||
$1
|
$1
|
||||||
|
|
@ -89,51 +89,51 @@ EOF
|
||||||
|
|
||||||
function log_small_title()
|
function log_small_title()
|
||||||
{
|
{
|
||||||
echo -e "\n${BOLD} > ${1}${NORMAL}\n"
|
echo -e "\n${BOLD} > ${1}${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function log_debug()
|
function log_debug()
|
||||||
{
|
{
|
||||||
echo "$1" >&3
|
echo "$1" >> /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_info()
|
function log_info()
|
||||||
{
|
{
|
||||||
echo "${1}"
|
echo "${1}" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_success()
|
function log_success()
|
||||||
{
|
{
|
||||||
echo "${BOLD}${GREEN}Success: ${1}${NORMAL}"
|
echo "${BOLD}${GREEN}Success: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_warning()
|
function log_warning()
|
||||||
{
|
{
|
||||||
echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}"
|
echo "${BOLD}${ORANGE}Warning: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_error()
|
function log_error()
|
||||||
{
|
{
|
||||||
echo "${BOLD}${RED}Error: ${1}${NORMAL}"
|
echo "${BOLD}${RED}Error: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_critical()
|
function log_critical()
|
||||||
{
|
{
|
||||||
echo "${BOLD}${RED}Critical: ${1}${NORMAL}"
|
echo "${BOLD}${RED}Critical: ${1}${NORMAL}" | tee -a /proc/self/fd/3
|
||||||
clean_exit 1
|
clean_exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_report_test_success () {
|
function log_report_test_success () {
|
||||||
echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n"
|
echo -e "\n${BOLD}${GREEN}--- SUCCESS ---${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_report_test_warning () {
|
function log_report_test_warning () {
|
||||||
echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n"
|
echo -e "\n${BOLD}${ORANGE}--- WARNING ---${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
function log_report_test_failed () {
|
function log_report_test_failed () {
|
||||||
echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n"
|
echo -e "\n${BOLD}${RED}--- FAIL ---${NORMAL}\n" | tee -a /proc/self/fd/3
|
||||||
}
|
}
|
||||||
|
|
||||||
#=================================================
|
#=================================================
|
||||||
|
|
|
||||||
56
lib/tests.sh
56
lib/tests.sh
|
|
@ -1,7 +1,7 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
#=================================================
|
#=================================================
|
||||||
# Logistic helpers
|
# "Low-level" logistic helpers
|
||||||
#=================================================
|
#=================================================
|
||||||
|
|
||||||
_RUN_YUNOHOST_CMD() {
|
_RUN_YUNOHOST_CMD() {
|
||||||
|
|
@ -258,13 +258,18 @@ PACKAGE_LINTER () {
|
||||||
# Execute package linter and linter_result gets the return code of the package linter
|
# Execute package linter and linter_result gets the return code of the package linter
|
||||||
./package_linter/package_linter.py "$package_path" | tee -a "$complete_log"
|
./package_linter/package_linter.py "$package_path" | tee -a "$complete_log"
|
||||||
./package_linter/package_linter.py "$package_path" --json | tee -a "$complete_log" > $current_test_results
|
./package_linter/package_linter.py "$package_path" --json | tee -a "$complete_log" > $current_test_results
|
||||||
|
|
||||||
|
return $?
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_INSTALL () {
|
TEST_INSTALL () {
|
||||||
# Try to install in a sub path, on root or without url access
|
|
||||||
# $1 = install type
|
|
||||||
|
|
||||||
local install_type=$1
|
local install_type=$1
|
||||||
|
|
||||||
|
# This is a separate case ... at least from an hystorical point of view ...
|
||||||
|
# but it helpers for semantic that the test is a "TEST_INSTALL" ...
|
||||||
|
[ "$install_type" = "multi" ] && { _TEST_MULTI_INSTANCE; return $?; }
|
||||||
|
|
||||||
local check_path="/"
|
local check_path="/"
|
||||||
local is_public="1"
|
local is_public="1"
|
||||||
[ "$install_type" = "subdir" ] && { start_test "Installation in a sub path"; local check_path=/path; }
|
[ "$install_type" = "subdir" ] && { start_test "Installation in a sub path"; local check_path=/path; }
|
||||||
|
|
@ -298,6 +303,29 @@ TEST_INSTALL () {
|
||||||
return $?
|
return $?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_TEST_MULTI_INSTANCE () {
|
||||||
|
|
||||||
|
start_test "Multi-instance installations"
|
||||||
|
|
||||||
|
# Check if an install have previously work
|
||||||
|
at_least_one_install_succeeded || return 1
|
||||||
|
|
||||||
|
local check_path=$(default_install_path)
|
||||||
|
|
||||||
|
LOAD_LXC_SNAPSHOT snap0
|
||||||
|
|
||||||
|
log_small_title "First installation: path=$DOMAIN$check_path" \
|
||||||
|
&&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \
|
||||||
|
&& log_small_title "Second installation: path=$SUBDOMAIN$check_path" \
|
||||||
|
&& _INSTALL_APP "path=$check_path" \
|
||||||
|
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \
|
||||||
|
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2 \
|
||||||
|
&& _REMOVE_APP \
|
||||||
|
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2
|
||||||
|
|
||||||
|
return $?
|
||||||
|
}
|
||||||
|
|
||||||
TEST_UPGRADE () {
|
TEST_UPGRADE () {
|
||||||
|
|
||||||
local commit=$1
|
local commit=$1
|
||||||
|
|
@ -351,27 +379,6 @@ TEST_UPGRADE () {
|
||||||
return $?
|
return $?
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_MULTI_INSTANCE () {
|
|
||||||
|
|
||||||
start_test "Multi-instance installations"
|
|
||||||
|
|
||||||
# Check if an install have previously work
|
|
||||||
at_least_one_install_succeeded || return 1
|
|
||||||
|
|
||||||
local check_path=$(default_install_path)
|
|
||||||
|
|
||||||
LOAD_LXC_SNAPSHOT snap0
|
|
||||||
|
|
||||||
log_small_title "First installation: path=$DOMAIN$check_path" \
|
|
||||||
&&_INSTALL_APP "domain=$DOMAIN" "path=$check_path" \
|
|
||||||
&& log_small_title "Second installation: path=$SUBDOMAIN$check_path" \
|
|
||||||
&&_INSTALL_APP "path=$check_path" \
|
|
||||||
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $DOMAIN $check_path \
|
|
||||||
&& _VALIDATE_THAT_APP_CAN_BE_ACCESSED $SUBDOMAIN $check_path "" ${app_id}__2
|
|
||||||
|
|
||||||
return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST_PORT_ALREADY_USED () {
|
TEST_PORT_ALREADY_USED () {
|
||||||
|
|
||||||
start_test "Port already used"
|
start_test "Port already used"
|
||||||
|
|
@ -464,6 +471,7 @@ TEST_BACKUP_RESTORE () {
|
||||||
|
|
||||||
# Place the copy of the backup archive in the container.
|
# Place the copy of the backup archive in the container.
|
||||||
sudo lxc file push -r ./ynh_backups $LXC_NAME/home/yunohost.backup/archives/
|
sudo lxc file push -r ./ynh_backups $LXC_NAME/home/yunohost.backup/archives/
|
||||||
|
RUN_INSIDE_LXC ls -l /home/yunohost.backup/archives/
|
||||||
|
|
||||||
log_small_title "Restore on a clean YunoHost system..."
|
log_small_title "Restore on a clean YunoHost system..."
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ source lib/witness.sh
|
||||||
complete_log="./Complete.log"
|
complete_log="./Complete.log"
|
||||||
|
|
||||||
# Purge some log files
|
# Purge some log files
|
||||||
> "$complete_log"
|
rm -f "$complete_log" && touch "$complete_log"
|
||||||
|
|
||||||
# Redirect fd 3 (=debug steam) to complete log
|
# Redirect fd 3 (=debug steam) to complete log
|
||||||
exec 3>>$complete_log
|
exec 3>>$complete_log
|
||||||
|
|
@ -16,9 +16,6 @@ exec 3>>$complete_log
|
||||||
# Parse the check_process and generate jsons that describe tests to run
|
# Parse the check_process and generate jsons that describe tests to run
|
||||||
#=======================================================================
|
#=======================================================================
|
||||||
|
|
||||||
# Parse the check_process only if it's exist
|
|
||||||
check_process="$package_path/check_process"
|
|
||||||
|
|
||||||
# Extract a section found between $1 and $2 from the file $3
|
# Extract a section found between $1 and $2 from the file $3
|
||||||
extract_check_process_section () {
|
extract_check_process_section () {
|
||||||
local source_file="${3:-$check_process}"
|
local source_file="${3:-$check_process}"
|
||||||
|
|
@ -156,11 +153,11 @@ parse_check_process() {
|
||||||
fi
|
fi
|
||||||
|
|
||||||
is_test_enabled pkg_linter && add_test "PACKAGE_LINTER"
|
is_test_enabled pkg_linter && add_test "PACKAGE_LINTER"
|
||||||
is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir"
|
|
||||||
is_test_enabled setup_root && add_test "TEST_INSTALL" "root"
|
is_test_enabled setup_root && add_test "TEST_INSTALL" "root"
|
||||||
|
is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir"
|
||||||
is_test_enabled setup_nourl && add_test "TEST_INSTALL" "nourl"
|
is_test_enabled setup_nourl && add_test "TEST_INSTALL" "nourl"
|
||||||
is_test_enabled setup_private && add_test "TEST_INSTALL" "private"
|
is_test_enabled setup_private && add_test "TEST_INSTALL" "private"
|
||||||
is_test_enabled multi_instance && add_test "TEST_MULTI_INSTANCE"
|
is_test_enabled multi_instance && add_test "TEST_INSTALL" "multi"
|
||||||
is_test_enabled backup_restore && add_test "TEST_BACKUP_RESTORE"
|
is_test_enabled backup_restore && add_test "TEST_BACKUP_RESTORE"
|
||||||
|
|
||||||
# Upgrades
|
# Upgrades
|
||||||
|
|
@ -220,15 +217,15 @@ guess_test_configuration() {
|
||||||
local install_args=$(python "./lib/manifest_parsing.py" "$package_path/manifest.json" | cut -d ':' -f1,2 | tr ':' '=' | tr '\n' '&')
|
local install_args=$(python "./lib/manifest_parsing.py" "$package_path/manifest.json" | cut -d ':' -f1,2 | tr ':' '=' | tr '\n' '&')
|
||||||
|
|
||||||
add_test "PACKAGE_LINTER"
|
add_test "PACKAGE_LINTER"
|
||||||
add_test "TEST_INSTALL subdir"
|
add_test "TEST_INSTALL" "root"
|
||||||
add_test "TEST_INSTALL root"
|
add_test "TEST_INSTALL" "subdir"
|
||||||
if echo $install_args | grep -q "is_public="
|
if echo $install_args | grep -q "is_public="
|
||||||
then
|
then
|
||||||
add_test "TEST_INSTALL" "private"
|
add_test "TEST_INSTALL" "private"
|
||||||
fi
|
fi
|
||||||
if grep multi_instance "$package_path/manifest.json" | grep -q true
|
if grep multi_instance "$package_path/manifest.json" | grep -q true
|
||||||
then
|
then
|
||||||
add_test "TEST_MULTI_INSTANCE"
|
add_test "TEST_INSTALL" "multi"
|
||||||
fi
|
fi
|
||||||
add_test "TEST_BACKUP_RESTORE"
|
add_test "TEST_BACKUP_RESTORE"
|
||||||
add_test "TEST_UPGRADE"
|
add_test "TEST_UPGRADE"
|
||||||
|
|
@ -243,6 +240,9 @@ run_all_tests() {
|
||||||
mkdir -p $TEST_CONTEXT/tests
|
mkdir -p $TEST_CONTEXT/tests
|
||||||
mkdir -p $TEST_CONTEXT/results
|
mkdir -p $TEST_CONTEXT/results
|
||||||
|
|
||||||
|
# Parse the check_process only if it's exist
|
||||||
|
check_process="$package_path/check_process"
|
||||||
|
|
||||||
[ -e "$check_process" ] \
|
[ -e "$check_process" ] \
|
||||||
&& parse_check_process \
|
&& parse_check_process \
|
||||||
|| guess_test_configuration
|
|| guess_test_configuration
|
||||||
|
|
@ -281,7 +281,7 @@ run_all_tests() {
|
||||||
done
|
done
|
||||||
|
|
||||||
# Print the final results of the tests
|
# Print the final results of the tests
|
||||||
# FIXME COMPUTE_RESULTS_SUMMARY $test_serie_id
|
python3 lib/analyze_test_results.py $TEST_CONTEXT 2>$TEST_CONTEXT/summary.json
|
||||||
|
|
||||||
# Restore the started time for the timer
|
# Restore the started time for the timer
|
||||||
starttime=$complete_start_timer
|
starttime=$complete_start_timer
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue