From fe2e0048adae4987ab9a64aacfa1644ee62d0a0e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 16 Jan 2023 20:38:56 +0100 Subject: [PATCH] Introduce a parse_tests_toml.py to interface with a modern replacement for horrendous check_process files --- lib/analyze_test_results.py | 18 ++-- lib/default_install_args.py | 1 - lib/parse_tests_toml.py | 163 ++++++++++++++++++++++++++++++++++++ lib/tests.sh | 2 +- lib/tests_coordination.sh | 4 +- 5 files changed, 175 insertions(+), 13 deletions(-) create mode 100644 lib/parse_tests_toml.py diff --git a/lib/analyze_test_results.py b/lib/analyze_test_results.py index 0da8c34..cc06dd7 100644 --- a/lib/analyze_test_results.py +++ b/lib/analyze_test_results.py @@ -25,10 +25,10 @@ def test_notes(test): if test["test_type"] == "TEST_UPGRADE" and test["test_arg"]: return - if test["test_type"] == "PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("warning"): + if test["test_type"] == "TEST_PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("warning"): yield '%s warnings' % len(test['results'].get("warning")) - if test["test_type"] == "PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("info"): + if test["test_type"] == "TEST_PACKAGE_LINTER" and test['results']['main_result'] == 'success' and test['results'].get("info"): yield '%s possible improvements' % len(set(test['results'].get("info"))) if test['results'].get("witness"): @@ -71,7 +71,7 @@ def level_1(tests): And there are no critical issues in the linter """ - linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"] install_tests = [t for t in tests if t["test_type"] == "TEST_INSTALL"] witness_missing_detected = any(t["results"].get("witness") for t in tests) @@ -126,7 +126,7 @@ def level_5(tests): """ alias_traversal_detected = any(t["results"].get("alias_traversal") for t in tests) - linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"] return not alias_traversal_detected \ and linter_tests != [] \ @@ -140,7 +140,7 @@ def level_6(tests): (the linter will report a warning named "is_in_github_org" if it's not) """ - linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"] return linter_tests != [] \ and "is_in_github_org" not in linter_tests[0]["results"]["warning"] @@ -153,7 +153,7 @@ def level_7(tests): linter which will report a "qualify_for_level_7" in successes) """ - linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"] # For runtime warnings, ignore stuff happening during upgrades from previous versions tests_on_which_to_check_for_runtime_warnings = [t for t in tests if not (t["test_type"] == "TEST_UPGRADE" and t["test_arg"])] @@ -178,7 +178,7 @@ def level_8(tests): which will report a "qualify_for_level_8") """ - linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"] return linter_tests != [] \ and "App.qualify_for_level_8" in linter_tests[0]["results"]["success"] @@ -190,7 +190,7 @@ def level_9(tests): App is flagged high-quality in the app catalog (this is tested by the linter which will rpeort a "qualify_for_level_9") """ - linter_tests = [t for t in tests if t["test_type"] == "PACKAGE_LINTER"] + linter_tests = [t for t in tests if t["test_type"] == "TEST_PACKAGE_LINTER"] return linter_tests != [] \ and "App.qualify_for_level_9" in linter_tests[0]["results"]["success"] @@ -199,7 +199,7 @@ def level_9(tests): def make_summary(): test_types = { - "PACKAGE_LINTER": "Package linter", + "TEST_PACKAGE_LINTER": "Package linter", "TEST_INSTALL": "Install", "TEST_UPGRADE": "Upgrade", "TEST_BACKUP_RESTORE": "Backup/restore", diff --git a/lib/default_install_args.py b/lib/default_install_args.py index 2c1e89b..8d6f1e6 100644 --- a/lib/default_install_args.py +++ b/lib/default_install_args.py @@ -41,7 +41,6 @@ def get_default_values_for_questions(manifest): else: raise Exception("No default value could be computed for arg " + name) - if __name__ == '__main__': manifest_path = sys.argv[1:][0] diff --git a/lib/parse_tests_toml.py b/lib/parse_tests_toml.py new file mode 100644 index 0000000..88e550b --- /dev/null +++ b/lib/parse_tests_toml.py @@ -0,0 +1,163 @@ +import sys +import os +import toml +import copy +import json + + +def generate_test_list_base(test_manifest, default_install_args, is_webapp, is_multi_instance): + + assert test_manifest["test_format"] == 1.0, "Only test_format 1.0 is supported for now" + + assert isinstance(test_manifest["default"], dict), "You should at least defined the 'default' test suite" + + is_full_domain_app = "domain" in default_install_args and "path" not in default_install_args + + for test_suite_id, test_suite in test_manifest.items(): + + # Ignore non-testsuite stuff like "test_format" + if not isinstance(test_suite, dict): + continue + + install_args = copy.copy(default_install_args) + install_args.update(test_suite.get("args")) + + default_meta = { + "preinstall_template": test_suite.get("preinstall_template", ""), + "preupgrade_template": test_suite.get("preupgrade_template", ""), + "install_args": install_args, + } + + yield test_suite_id, "package_linter", default_meta + + if is_webapp: + yield test_suite_id, "install.root", default_meta + if not is_full_domain_app: + yield test_suite_id, "install.subdir", default_meta + else: + yield test_suite_id, "install.nourl", default_meta + + if is_webapp and ("is_public" in install_args or "init_main_permission" in install_args): + yield test_suite_id, "install.private", default_meta + + if is_multi_instance: + yield test_suite_id, "install.multi", default_meta + + yield test_suite_id, "backup_restore", default_meta + + yield test_suite_id, "upgrade", default_meta + for commit, infos in test_suite.get("test_upgrade_from", {}).items(): + upgrade_meta = copy.copy(default_meta) + upgrade_meta.update(infos) + yield test_suite_id, "upgrade." + commit, upgrade_meta + + if is_webapp: + yield test_suite_id, "change_url", default_meta + + +def filter_test_list(test_manifest, base_test_list): + + for test_suite_id, test_suite in test_manifest.items(): + + # Ignore non-testsuite stuff like "test_format" + if not isinstance(test_suite, dict): + continue + + exclude = test_suite.get("exclude", []) + only = test_suite.get("only") + + if test_suite_id == "default" and only: + raise Exception("'only' is not allowed on the default test suite") + + if only: + tests_for_this_suite = {test_id: meta + for suite_id, test_id, meta in base_test_list + if suite_id == test_suite_id and test_id in only} + elif exclude: + tests_for_this_suite = {test_id: meta + for suite_id, test_id, meta in base_test_list + if suite_id == test_suite_id and test_id not in exclude} + else: + tests_for_this_suite = {test_id: meta + for suite_id, test_id, meta in base_test_list + if suite_id == test_suite_id} + + yield test_suite_id, tests_for_this_suite + + +def dump_for_package_check(test_list, package_check_tests_dir): + + test_suite_i = 0 + + for test_suite_id, subtest_list in test_list.items(): + + test_suite_i += 1 + + subtest_i = 0 + + for test, meta in subtest_list.items(): + + meta = copy.copy(meta) + + subtest_i += 1 + + if "." in test: + test_type, test_arg = test.split(".") + else: + test_type = test + test_arg = "" + + J = { + "test_serie": test_suite_id, + "test_type": "TEST_" + test_type.upper(), + "test_arg": test_arg, + "preinstall_template": meta.pop("preinstall_template", ""), + "preupgrade_template": meta.pop("preupgrade_template", ""), + "install_args": '&'.join([k + "=" + str(v) for k, v in meta.pop("install_args").items()]), + "extra": meta # Boring logic just to ship the upgrade-from-commit's name ... + } + + test_file_id = test_suite_i * 100 + subtest_i + + json.dump(J, open(package_check_tests_dir + f"/{test_file_id}.json", "w")) + + +def build_test_list(basedir): + + test_manifest = toml.load(open(basedir + "/tests.toml", "r")) + + if os.path.exists(basedir + "/manifest.json"): + manifest = json.load(open(basedir + "/manifest.json", "r")) + is_multi_instance = manifest.get("multi_instance") is True + else: + manifest = toml.load(open(basedir + "/manifest.toml", "r")) + is_multi_instance = manifest.get("integration").get("multi_instance") is True + + is_webapp = os.system(f"grep -q '^ynh_add_nginx_config' '{basedir}/scripts/install'") == 0 + + from default_install_args import get_default_values_for_questions + default_install_args = {k: v for k, v in get_default_values_for_questions(manifest)} + + base_test_list = list(generate_test_list_base(test_manifest, default_install_args, is_webapp, is_multi_instance)) + test_list = {test_suite_id: tests for test_suite_id, tests in filter_test_list(test_manifest, base_test_list)} + + return test_list + +if __name__ == '__main__': + + test_list = build_test_list(sys.argv[1]) + + if len(sys.argv) <= 1 or sys.argv[1] in ["-h", "--help"]: + print("""Usage: + +Display generated test list: + python3 parse_tests_toml.py /path/to/app/folder/ | jq + +Dump the list (only relevant from inside package_checker's code ...) + python3 parse_tests_toml.py /path/to/app/folder/ /tmp/dir/for/package/check/.... + +""") + elif len(sys.argv) == 2: + print(json.dumps(test_list, indent=4)) + else: + dump_for_package_check(test_list, sys.argv[2]) diff --git a/lib/tests.sh b/lib/tests.sh index 2ed58fe..ecd16a9 100644 --- a/lib/tests.sh +++ b/lib/tests.sh @@ -350,7 +350,7 @@ Page extract:\n$page_extract" > $TEST_CONTEXT/curl_result # Tests #================================================= -PACKAGE_LINTER () { +TEST_PACKAGE_LINTER () { start_test "Package linter" diff --git a/lib/tests_coordination.sh b/lib/tests_coordination.sh index 6c4271e..c216b60 100644 --- a/lib/tests_coordination.sh +++ b/lib/tests_coordination.sh @@ -133,7 +133,7 @@ parse_check_process() { test_serie=${tests_serie//;; } - is_test_enabled pkg_linter && add_test "PACKAGE_LINTER" + is_test_enabled pkg_linter && add_test "TEST_PACKAGE_LINTER" is_test_enabled setup_root && add_test "TEST_INSTALL" "root" is_test_enabled setup_sub_dir && add_test "TEST_INSTALL" "subdir" is_test_enabled setup_nourl && add_test "TEST_INSTALL" "nourl" @@ -196,7 +196,7 @@ guess_test_configuration() { local install_args=$(python3 "./lib/default_install_args.py" "$package_path"/manifest.*) - add_test "PACKAGE_LINTER" + add_test "TEST_PACKAGE_LINTER" add_test "TEST_INSTALL" "root" add_test "TEST_INSTALL" "subdir" if echo $install_args | grep -q "is_public=\|init_main_permission="