mirror of
https://github.com/YunoHost/package_check.git
synced 2024-09-03 20:06:20 +02:00
235 lines
6.9 KiB
Python
235 lines
6.9 KiB
Python
import sys
|
|
import json
|
|
import os
|
|
import time
|
|
|
|
def load_tests(test_folder):
|
|
|
|
for test in sorted(os.listdir(test_folder + "/tests")):
|
|
|
|
j = json.load(open(test_folder + "/tests/" + test))
|
|
j["id"] = os.path.basename(test).split(".")[0]
|
|
j["results"] = json.load(open(test_folder + "/results/" + j["id"] + ".json"))
|
|
yield j
|
|
|
|
|
|
levels = []
|
|
|
|
|
|
def level(level_, descr):
|
|
def decorator(f):
|
|
f.descr = descr
|
|
f.level = level_
|
|
levels.insert(level_, f)
|
|
return f
|
|
return decorator
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
|
@level(0, "Broken")
|
|
def level_0(tests):
|
|
return True
|
|
|
|
|
|
@level(1, "Installable in at least one scenario")
|
|
def level_1(tests):
|
|
"""
|
|
Any install test succeded
|
|
And there are no critical issues in the linter
|
|
"""
|
|
|
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
|
install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
|
|
witness_missing_detected = any(t["results"].get("witness") for t in tests)
|
|
|
|
return linter_tests != [] \
|
|
and linter_tests[0]["results"]["critical"] == [] \
|
|
and not witness_missing_detected \
|
|
and any(t["results"]["main_result"] == "success" for t in install_tests)
|
|
|
|
|
|
@level(2, "Installable in all scenarios")
|
|
def level_2(tests):
|
|
"""
|
|
All install tests succeeded (and at least one test was made)
|
|
"""
|
|
|
|
install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
|
|
|
|
return install_tests != [] \
|
|
and all(t["results"]["main_result"] == "success" for t in install_tests)
|
|
|
|
|
|
@level(3, "Can be upgraded")
|
|
def level_3(tests):
|
|
"""
|
|
All upgrade tests succeeded (and at least one test was made)
|
|
"""
|
|
|
|
upgrade_same_version_tests = [t for t in tests if t["test_type"] == "TEST_UPGRADE" and not t["test_arg"]]
|
|
|
|
return upgrade_same_version_tests != [] \
|
|
and all(t["results"]["main_result"] == "success" for t in upgrade_same_version_tests)
|
|
|
|
|
|
@level(4, "Can be backup/restored")
|
|
def level_4(tests):
|
|
"""
|
|
All backup/restore tests succeded (and at least one test was made)
|
|
"""
|
|
|
|
backup_tests = [t for t in tests if t["test_type"] == "TEST_BACKUP_RESTORE"]
|
|
|
|
return backup_tests != [] \
|
|
and all(t["results"]["main_result"] == "success" for t in backup_tests)
|
|
|
|
|
|
@level(5, "No linter errors")
|
|
def level_5(tests):
|
|
"""
|
|
Linter returned no errors (= main_result is success)
|
|
and no alias/path traversal issue detected during tests
|
|
and not too many warnings in log outputs
|
|
"""
|
|
|
|
alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests)
|
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
|
|
|
return not alias_traversal_detected \
|
|
and linter_tests != [] \
|
|
and linter_tests[0]["results"]["main_result"] == "success"
|
|
|
|
|
|
@level(6, "App is in a community-operated git org")
|
|
def level_6(tests):
|
|
"""
|
|
The app is in the Yunohost-Apps organization
|
|
(the linter will report a warning named "is_in_github_org" if it's not)
|
|
"""
|
|
|
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
|
|
|
return linter_tests != [] \
|
|
and "is_in_github_org" not in linter_tests[0]["results"]["warning"]
|
|
|
|
|
|
@level(7, "Pass all tests + no linter warnings")
|
|
def level_7(tests):
|
|
"""
|
|
All tests succeeded + no warning in linter (that part is tested by the
|
|
linter which will report a "qualify_for_level_7" in successes)
|
|
"""
|
|
|
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
|
too_many_warnings = any(t["results"].get("too_many_warnings") for t in tests)
|
|
|
|
return all(t["results"]["main_result"] == "success" for t in tests) \
|
|
and linter_tests != [] \
|
|
and not too_many_warnings \
|
|
and "App.qualify_for_level_7" in linter_tests[0]["results"]["success"]
|
|
|
|
|
|
@level(8, "Maintained and long-term good quality")
|
|
def level_8(tests):
|
|
"""
|
|
App is maintained and long-term good quality (this is tested by the linter
|
|
which will report a "qualify_for_level_8")
|
|
"""
|
|
|
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
|
|
|
return linter_tests != [] \
|
|
and "App.qualify_for_level_8" in linter_tests[0]["results"]["success"]
|
|
|
|
|
|
@level(9, "Flagged high-quality in app catalog")
|
|
def level_9(tests):
|
|
"""
|
|
App is flagged high-quality in the app catalog (this is tested by the linter
|
|
which will rpeort a "qualify_for_level_9")
|
|
"""
|
|
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"]
|
|
|
|
return linter_tests != [] \
|
|
and "App.qualify_for_level_9" in linter_tests[0]["results"]["success"]
|
|
|
|
|
|
test_context = sys.argv[1]
|
|
tests = list(load_tests(test_context))
|
|
|
|
test_types = {
|
|
"PACKAGE_LINTER": "Package linter",
|
|
"TEST_INSTALL": "Install",
|
|
"TEST_UPGRADE": "Upgrade",
|
|
"TEST_BACKUP_RESTORE": "Backup/restore",
|
|
"TEST_CHANGE_URL": "Change url",
|
|
"TEST_PORT_ALREADY_USED": "Port already used",
|
|
"ACTIONS_CONFIG_PANEL": "Config/panel"
|
|
}
|
|
|
|
OK = ' \033[1m\033[92mOK\033[0m '
|
|
FAIL = '\033[91mfail\033[0m'
|
|
|
|
latest_test_serie = "default"
|
|
print()
|
|
for test in tests:
|
|
test_display_name = test_types[test["test_type"]]
|
|
if test["test_arg"]:
|
|
test_display_name += " (%s)" % test["test_arg"][:8]
|
|
test_display_name += ":"
|
|
if test["test_serie"] != latest_test_serie:
|
|
latest_test_serie = test["test_serie"]
|
|
print("------------- %s -------------" % latest_test_serie)
|
|
|
|
result = OK if test["results"]["main_result"] == "success" else FAIL
|
|
print("{test: <30}{result}".format(test=test_display_name, result=result))
|
|
|
|
print()
|
|
print("Level results")
|
|
print("=============")
|
|
|
|
stop_global_level_bump = False
|
|
|
|
global_level = level_0
|
|
|
|
for level in levels[1:]:
|
|
level.passed = level(tests)
|
|
|
|
if not level.passed:
|
|
stop_global_level_bump = True
|
|
|
|
if not stop_global_level_bump:
|
|
global_level = level
|
|
display = OK
|
|
else:
|
|
display = " ok " if level.passed else ""
|
|
|
|
print("Level {i} {descr: <40} {result}".format(i=level.level,
|
|
descr="(%s)"%level.descr[:38], result=display))
|
|
|
|
print()
|
|
print("\033[1mGlobal level for this application: %s (%s)\033[0m" % (global_level.level, global_level.descr))
|
|
print()
|
|
|
|
|
|
summary = {
|
|
"app": open(test_context + "/app_id").read().strip(),
|
|
"commit": open(test_context + "/commit").read().strip(),
|
|
"architecture": open(test_context + "/architecture").read().strip(),
|
|
"yunohost_version": open(test_context + "/ynh_version").read().strip(),
|
|
"yunohost_branch": open(test_context + "/ynh_branch").read().strip(),
|
|
"timestamp": int(time.time()),
|
|
"tests": [{
|
|
"test_type": t["test_type"],
|
|
"test_arg": t["test_arg"],
|
|
"test_serie": t["test_serie"],
|
|
"main_result": t["results"]["main_result"],
|
|
"test_duration": t["results"]["test_duration"]
|
|
} for t in tests],
|
|
"level_results": {level.level: level.passed for level in levels[1:]},
|
|
"level": global_level.level
|
|
}
|
|
|
|
sys.stderr.write(json.dumps(summary, indent=4))
|