Introduce a parse_tests_toml.py to interface with a modern replacement for horrendous check_process files

This commit is contained in:
Alexandre Aubin 2023-01-16 20:38:56 +01:00
parent 5e821aee23
commit fe2e0048ad
5 changed files with 175 additions and 13 deletions

View file

@ -25,10 +25,10 @@ def test_notes(test):
if test["test_type"] == "TEST_UPGRADE" and test["test_arg"]: if test["test_type"] == "TEST_UPGRADE" and test["test_arg"]:
return return
if test["test_type"] == "PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("warning"): if test["test_type"] == "TEST_PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("warning"):
yield '<style=warning>%s warnings</style>' % len(test['results'].get("warning")) yield '<style=warning>%s warnings</style>' % len(test['results'].get("warning"))
if test["test_type"] == "PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("info"): if test["test_type"] == "TEST_PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("info"):
yield '%s possible improvements' % len(set(test['results'].get("info"))) yield '%s possible improvements' % len(set(test['results'].get("info")))
if test['results'].get("witness"): if test['results'].get("witness"):
@ -71,7 +71,7 @@ def level_1(tests):
And there are no critical issues in the linter And there are no critical issues in the linter
""" """
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"]
install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"] install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"]
witness_missing_detected = any(t["results"].get("witness") for t in tests) witness_missing_detected = any(t["results"].get("witness") for t in tests)
@ -126,7 +126,7 @@ def level_5(tests):
""" """
alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests) alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests)
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"]
return not alias_traversal_detected \ return not alias_traversal_detected \
and linter_tests != [] \ and linter_tests != [] \
@ -140,7 +140,7 @@ def level_6(tests):
(the linter will report a warning named "is_in_github_org" if it's not) (the linter will report a warning named "is_in_github_org" if it's not)
""" """
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"]
return linter_tests != [] \ return linter_tests != [] \
and "is_in_github_org" not in linter_tests[0]["results"]["warning"] and "is_in_github_org" not in linter_tests[0]["results"]["warning"]
@ -153,7 +153,7 @@ def level_7(tests):
linter which will report a "qualify_for_level_7" in successes) linter which will report a "qualify_for_level_7" in successes)
""" """
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"]
# For runtime warnings, ignore stuff happening during upgrades from previous versions # For runtime warnings, ignore stuff happening during upgrades from previous versions
tests_on_which_to_check_for_runtime_warnings = [t for t in tests if not (t["test_type"] == "TEST_UPGRADE" and t["test_arg"])] tests_on_which_to_check_for_runtime_warnings = [t for t in tests if not (t["test_type"] == "TEST_UPGRADE" and t["test_arg"])]
@ -178,7 +178,7 @@ def level_8(tests):
which will report a "qualify_for_level_8") which will report a "qualify_for_level_8")
""" """
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"]
return linter_tests != [] \ return linter_tests != [] \
and "App.qualify_for_level_8" in linter_tests[0]["results"]["success"] and "App.qualify_for_level_8" in linter_tests[0]["results"]["success"]
@ -190,7 +190,7 @@ def level_9(tests):
App is flagged high-quality in the app catalog (this is tested by the linter App is flagged high-quality in the app catalog (this is tested by the linter
which will rpeort a "qualify_for_level_9") which will rpeort a "qualify_for_level_9")
""" """
linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"]
return linter_tests != [] \ return linter_tests != [] \
and "App.qualify_for_level_9" in linter_tests[0]["results"]["success"] and "App.qualify_for_level_9" in linter_tests[0]["results"]["success"]
@ -199,7 +199,7 @@ def level_9(tests):
def make_summary(): def make_summary():
test_types = { test_types = {
"PACKAGE_LINTER": "Package linter", "TEST_PACKAGE_LINTER": "Package linter",
"TEST_INSTALL": "Install", "TEST_INSTALL": "Install",
"TEST_UPGRADE": "Upgrade", "TEST_UPGRADE": "Upgrade",
"TEST_BACKUP_RESTORE": "Backup/restore", "TEST_BACKUP_RESTORE": "Backup/restore",

View file

@ -41,7 +41,6 @@ def get_default_values_for_questions(manifest):
else: else:
raise Exception("No default value could be computed for arg " + name) raise Exception("No default value could be computed for arg " + name)
if __name__ == '__main__': if __name__ == '__main__':
manifest_path = sys.argv[1:][0] manifest_path = sys.argv[1:][0]

163
lib/parse_tests_toml.py Normal file
View file

@ -0,0 +1,163 @@
import sys
import os
import toml
import copy
import json
def generate_test_list_base(test_manifest, default_install_args, is_webapp, is_multi_instance):
assert test_manifest["test_format"] == 1.0, "Only test_format 1.0 is supported for now"
assert isinstance(test_manifest["default"], dict), "You should at least defined the 'default' test suite"
is_full_domain_app = "domain" in default_install_args and "path" not in default_install_args
for test_suite_id, test_suite in test_manifest.items():
# Ignore non-testsuite stuff like "test_format"
if not isinstance(test_suite, dict):
continue
install_args = copy.copy(default_install_args)
install_args.update(test_suite.get("args"))
default_meta = {
"preinstall_template": test_suite.get("preinstall_template", ""),
"preupgrade_template": test_suite.get("preupgrade_template", ""),
"install_args": install_args,
}
yield test_suite_id, "package_linter", default_meta
if is_webapp:
yield test_suite_id, "install.root", default_meta
if not is_full_domain_app:
yield test_suite_id, "install.subdir", default_meta
else:
yield test_suite_id, "install.nourl", default_meta
if is_webapp and ("is_public" in install_args or "init_main_permission" in install_args):
yield test_suite_id, "install.private", default_meta
if is_multi_instance:
yield test_suite_id, "install.multi", default_meta
yield test_suite_id, "backup_restore", default_meta
yield test_suite_id, "upgrade", default_meta
for commit, infos in test_suite.get("test_upgrade_from", {}).items():
upgrade_meta = copy.copy(default_meta)
upgrade_meta.update(infos)
yield test_suite_id, "upgrade." + commit, upgrade_meta
if is_webapp:
yield test_suite_id, "change_url", default_meta
def filter_test_list(test_manifest, base_test_list):
for test_suite_id, test_suite in test_manifest.items():
# Ignore non-testsuite stuff like "test_format"
if not isinstance(test_suite, dict):
continue
exclude = test_suite.get("exclude", [])
only = test_suite.get("only")
if test_suite_id == "default" and only:
raise Exception("'only' is not allowed on the default test suite")
if only:
tests_for_this_suite = {test_id: meta
for suite_id, test_id, meta in base_test_list
if suite_id == test_suite_id and test_id in only}
elif exclude:
tests_for_this_suite = {test_id: meta
for suite_id, test_id, meta in base_test_list
if suite_id == test_suite_id and test_id not in exclude}
else:
tests_for_this_suite = {test_id: meta
for suite_id, test_id, meta in base_test_list
if suite_id == test_suite_id}
yield test_suite_id, tests_for_this_suite
def dump_for_package_check(test_list, package_check_tests_dir):
test_suite_i = 0
for test_suite_id, subtest_list in test_list.items():
test_suite_i += 1
subtest_i = 0
for test, meta in subtest_list.items():
meta = copy.copy(meta)
subtest_i += 1
if "." in test:
test_type, test_arg = test.split(".")
else:
test_type = test
test_arg = ""
J = {
"test_serie": test_suite_id,
"test_type": "TEST_" + test_type.upper(),
"test_arg": test_arg,
"preinstall_template": meta.pop("preinstall_template", ""),
"preupgrade_template": meta.pop("preupgrade_template", ""),
"install_args": '&'.join([k + "=" + str(v) for k, v in meta.pop("install_args").items()]),
"extra": meta # Boring logic just to ship the upgrade-from-commit's name ...
}
test_file_id = test_suite_i * 100 + subtest_i
json.dump(J, open(package_check_tests_dir + f"/{test_file_id}.json", "w"))
def build_test_list(basedir):
test_manifest = toml.load(open(basedir + "/tests.toml", "r"))
if os.path.exists(basedir + "/manifest.json"):
manifest = json.load(open(basedir + "/manifest.json", "r"))
is_multi_instance = manifest.get("multi_instance") is True
else:
manifest = toml.load(open(basedir + "/manifest.toml", "r"))
is_multi_instance = manifest.get("integration").get("multi_instance") is True
is_webapp = os.system(f"grep -q '^ynh_add_nginx_config' '{basedir}/scripts/install'") == 0
from default_install_args import get_default_values_for_questions
default_install_args = {k: v for k, v in get_default_values_for_questions(manifest)}
base_test_list = list(generate_test_list_base(test_manifest, default_install_args, is_webapp, is_multi_instance))
test_list = {test_suite_id: tests for test_suite_id, tests in filter_test_list(test_manifest, base_test_list)}
return test_list
if __name__ == '__main__':
test_list = build_test_list(sys.argv[1])
if len(sys.argv) <= 1 or sys.argv[1] in ["-h", "--help"]:
print("""Usage:
Display generated test list:
python3 parse_tests_toml.py /path/to/app/folder/ | jq
Dump the list (only relevant from inside package_checker's code ...)
python3 parse_tests_toml.py /path/to/app/folder/ /tmp/dir/for/package/check/....
""")
elif len(sys.argv) == 2:
print(json.dumps(test_list, indent=4))
else:
dump_for_package_check(test_list, sys.argv[2])

View file

@ -350,7 +350,7 @@ Page extract:\n$page_extract" > $TEST_CONTEXT/curl_result
# Tests # Tests
#================================================= #=================================================
PACKAGE_LINTER () { TEST_PACKAGE_LINTER () {
start_test "Package linter" start_test "Package linter"

View file

@ -133,7 +133,7 @@ parse_check_process() {
test_serie=${tests_serie//;; } test_serie=${tests_serie//;; }
is_test_enabled pkg_linter && add_test "PACKAGE_LINTER" is_test_enabled pkg_linter && add_test "TEST_PACKAGE_LINTER"
is_test_enabled setup_root && add_test "TEST_INSTALL" "root" is_test_enabled setup_root && add_test "TEST_INSTALL" "root"
is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir" is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir"
is_test_enabled setup_nourl && add_test "TEST_INSTALL" "nourl" is_test_enabled setup_nourl && add_test "TEST_INSTALL" "nourl"
@ -196,7 +196,7 @@ guess_test_configuration() {
local install_args=$(python3 "./lib/default_install_args.py" "$package_path"/manifest.*) local install_args=$(python3 "./lib/default_install_args.py" "$package_path"/manifest.*)
add_test "PACKAGE_LINTER" add_test "TEST_PACKAGE_LINTER"
add_test "TEST_INSTALL" "root" add_test "TEST_INSTALL" "root"
add_test "TEST_INSTALL" "subdir" add_test "TEST_INSTALL" "subdir"
if echo $install_args | grep -q "is_public=\|init_main_permission=" if echo $install_args | grep -q "is_public=\|init_main_permission="