From d24e727a74ebbdeb73ffbba2359b6414fb1d8d29 Mon Sep 17 00:00:00 2001 From: axolotle Date: Sat, 19 Nov 2022 17:39:30 +0100 Subject: [PATCH 1/3] [enh] add custom catalog builder --- custom-catalog/catalog_builder.py | 98 +++++++++++++++++++++++++++++++ custom-catalog/catalog_list.py | 27 +++++++++ ynh-dev | 39 ++++++++++++ 3 files changed, 164 insertions(+) create mode 100755 custom-catalog/catalog_builder.py create mode 100644 custom-catalog/catalog_list.py diff --git a/custom-catalog/catalog_builder.py b/custom-catalog/catalog_builder.py new file mode 100755 index 0000000..64d7f11 --- /dev/null +++ b/custom-catalog/catalog_builder.py @@ -0,0 +1,98 @@ +#!/usr/bin/python3 + +import os +import json +import toml +import shutil +import time +import subprocess +from collections import OrderedDict + + +now = time.time() +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" + +DEFAULT_APPS_FOLDER = "/ynh-dev/custom-apps/" +DEFAULT_APP_BRANCH = "master" + + +def build(folder=DEFAULT_APPS_FOLDER): + assert os.path.exists(folder), f"'{folder}' doesn't exists." + + app_list_path = os.path.join(folder, "apps.json") + assert os.path.exists(app_list_path), "no 'apps.json' app list found." + + with open(app_list_path) as f: + app_list = json.load(f) + + apps = {} + + for app, infos in app_list.items(): + app = app.lower() + try: + app_dict = build_app_dict(app, infos, folder) + except Exception as e: + print(f"Processing {app} failed: {str(e)}") + continue + + apps[app_dict["id"]] = app_dict + + # We also remove the app install question and resources parts which aint needed anymore by webadmin etc (or at least we think ;P) + for app in apps.values(): + if "manifest" in app and "install" in app["manifest"]: + del app["manifest"]["install"] + if "manifest" in app and "resources" in app["manifest"]: + del app["manifest"]["resources"] + + output_file = os.path.join(folder, "catalog.json") + data = { + "apps": apps, + "from_api_version": 3, + } + + with open(output_file, "w") as f: + f.write(json.dumps(data, sort_keys=True, indent=2)) + + +def build_app_dict(app, infos, folder): + app_folder = os.path.join(folder, app + "_ynh") + + # Build the dict with all the infos + manifest_toml = os.path.join(app_folder, "manifest.toml") + manifest_json = os.path.join(app_folder, "manifest.json") + if os.path.exists(manifest_toml): + with open(manifest_toml) as f: + manifest = toml.load(f, _dict=OrderedDict) + else: + with open(manifest_json) as f: + manifest = json.load(f, _dict=OrderedDict) + + return { + "id": app, + "git": { + "branch": infos.get("branch", DEFAULT_APP_BRANCH), + "revision": infos.get("revision", "HEAD"), + "url": f"file://{app_folder}", + }, + "lastUpdate": now, + "manifest": manifest, + "state": infos.get("state", "notworking"), + "level": infos.get("level", -1), + "maintained": infos.get("maintained", True), + # "high_quality": infos.get("high_quality", False), + # "featured": infos.get("featured", False), + "category": infos.get("category", None), + "subtags": infos.get("subtags", []), + "potential_alternative_to": infos.get("potential_alternative_to", []), + "antifeatures": list( + set( + list(manifest.get("antifeatures", {}).keys()) + + infos.get("antifeatures", []) + ) + ), + } + + +if __name__ == "__main__": + build() diff --git a/custom-catalog/catalog_list.py b/custom-catalog/catalog_list.py new file mode 100644 index 0000000..85b1a07 --- /dev/null +++ b/custom-catalog/catalog_list.py @@ -0,0 +1,27 @@ +import os +import yaml + +CATALOG_LIST_PATH = "/etc/yunohost/apps_catalog.yml" +assert os.path.exists(CATALOG_LIST_PATH), f"Catalog list yaml file '{CATALOG_LIST_PATH} does not exists" + + +def reset(): + with open(CATALOG_LIST_PATH, "w") as f: + catalog_list = [{"id": "default", "url": "https://app.yunohost.org/default/"}] + yaml.safe_dump(catalog_list, f, default_flow_style=False) + + +def add(): + with open(CATALOG_LIST_PATH) as f: + catalog_list = yaml.load(f, Loader=yaml.FullLoader) + ids = [catalog["id"] for catalog in catalog_list] + if "custom" not in ids: + catalog_list.append({"id": "custom", "url": None}) + with open(CATALOG_LIST_PATH, "w") as f: + yaml.safe_dump(catalog_list, f, default_flow_style=False) + + +def override(): + with open(CATALOG_LIST_PATH, "w") as f: + catalog_list = [{"id": "custom", "url": None}] + yaml.safe_dump(catalog_list, f, default_flow_style=False) diff --git a/ynh-dev b/ynh-dev index bfca9f7..9871bbd 100755 --- a/ynh-dev +++ b/ynh-dev @@ -43,6 +43,8 @@ function main() use-git|--use-git) use_git $ARGUMENTS ;; test|--test) run_tests $ARGUMENTS ;; + catalog|--catalog) catalog $ARGUMENTS ;; + *) critical "Unknown action ${ACTION}." ;; esac } @@ -483,4 +485,41 @@ function run_tests() done } +function catalog() +{ + assert_inside_vm + local ACTION="$1" + local CUSTOM_APPS_FOLDER=${2:-"/ynh-dev/custom-apps"} + local CUSTOM_CAT_PATH="${CUSTOM_APPS_FOLDER}/catalog.json" + local CACHE_FOLDER="/var/cache/yunohost/repo" + + cd /ynh-dev/custom-catalog/ + + case "${ACTION}" in + build) + info "Rebuilding custom app catalog" + python3 -c "from catalog_builder import build; build(folder='${CUSTOM_APPS_FOLDER}')" && success "Successfully build custom catalog list in '${CUSTOM_CAT_PATH}'" + ;; + add) + info "Injecting custom catalog in YunoHost catalog list" + create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json" + python3 -c "from catalog_list import add; add()" && success "Custom catalog '${CUSTOM_CAT_PATH}' added to catalog list" + ;; + override) + info "Overriding default catalog by custom one" + create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json" + python3 -c "from catalog_list import override; override()" && success "Default catalog is now overrided by '$CUSTOM_CAT_PATH'" + ;; + reset) + info "Reseting to YunoHost default catalog list" + [ -e "$CACHE_FOLDER/custom.json" ] && rm "$CACHE_FOLDER/custom.json" + python3 -c "from catalog_list import reset; reset()" || exit 1 + success "Returned to default" + ;; + *) + critical "Unknown catalog action '${ACTION}'." + ;; + esac +} + main $@ From e1d5b6e0117f850156b6f9268f4fa37d0ec7f622 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 19 Nov 2022 21:45:38 +0100 Subject: [PATCH 2/3] Simplify new custom-catalog code and flow --- custom-catalog/apps.json | 15 ++++++++ custom-catalog/catalog_list.py | 27 --------------- ...{catalog_builder.py => catalog_manager.py} | 34 +++++++++++++++---- ynh-dev | 16 +++++---- 4 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 custom-catalog/apps.json delete mode 100644 custom-catalog/catalog_list.py rename custom-catalog/{catalog_builder.py => catalog_manager.py} (70%) diff --git a/custom-catalog/apps.json b/custom-catalog/apps.json new file mode 100644 index 0000000..5d19da0 --- /dev/null +++ b/custom-catalog/apps.json @@ -0,0 +1,15 @@ +{ + "0testapp": { + "antifeatures": [ + "non-free-assets" + ], + "branch": "master", + "category": "games", + "level": 8, + "potential_alternative_to": [ + "Google Agenda", + "Microsoft Outlook" + ], + "state": "working" + } +} diff --git a/custom-catalog/catalog_list.py b/custom-catalog/catalog_list.py deleted file mode 100644 index 85b1a07..0000000 --- a/custom-catalog/catalog_list.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import yaml - -CATALOG_LIST_PATH = "/etc/yunohost/apps_catalog.yml" -assert os.path.exists(CATALOG_LIST_PATH), f"Catalog list yaml file '{CATALOG_LIST_PATH} does not exists" - - -def reset(): - with open(CATALOG_LIST_PATH, "w") as f: - catalog_list = [{"id": "default", "url": "https://app.yunohost.org/default/"}] - yaml.safe_dump(catalog_list, f, default_flow_style=False) - - -def add(): - with open(CATALOG_LIST_PATH) as f: - catalog_list = yaml.load(f, Loader=yaml.FullLoader) - ids = [catalog["id"] for catalog in catalog_list] - if "custom" not in ids: - catalog_list.append({"id": "custom", "url": None}) - with open(CATALOG_LIST_PATH, "w") as f: - yaml.safe_dump(catalog_list, f, default_flow_style=False) - - -def override(): - with open(CATALOG_LIST_PATH, "w") as f: - catalog_list = [{"id": "custom", "url": None}] - yaml.safe_dump(catalog_list, f, default_flow_style=False) diff --git a/custom-catalog/catalog_builder.py b/custom-catalog/catalog_manager.py similarity index 70% rename from custom-catalog/catalog_builder.py rename to custom-catalog/catalog_manager.py index 64d7f11..a05bbd4 100755 --- a/custom-catalog/catalog_builder.py +++ b/custom-catalog/catalog_manager.py @@ -1,19 +1,21 @@ #!/usr/bin/python3 +import sys import os import json import toml -import shutil +import yaml import time -import subprocess from collections import OrderedDict +CATALOG_LIST_PATH = "/etc/yunohost/apps_catalog.yml" +assert os.path.exists(CATALOG_LIST_PATH), f"Catalog list yaml file '{CATALOG_LIST_PATH} does not exists" now = time.time() my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" -DEFAULT_APPS_FOLDER = "/ynh-dev/custom-apps/" +DEFAULT_APPS_FOLDER = "/ynh-dev/custom-catalog/" DEFAULT_APP_BRANCH = "master" @@ -27,13 +29,15 @@ def build(folder=DEFAULT_APPS_FOLDER): app_list = json.load(f) apps = {} + fail = False for app, infos in app_list.items(): app = app.lower() try: app_dict = build_app_dict(app, infos, folder) except Exception as e: - print(f"Processing {app} failed: {str(e)}") + print(f"[\033[1m\033[31mFAIL\033[00m] Processing {app} failed: {str(e)}") + fail = True continue apps[app_dict["id"]] = app_dict @@ -54,6 +58,8 @@ def build(folder=DEFAULT_APPS_FOLDER): with open(output_file, "w") as f: f.write(json.dumps(data, sort_keys=True, indent=2)) + if fail: + sys.exit(1) def build_app_dict(app, infos, folder): app_folder = os.path.join(folder, app + "_ynh") @@ -93,6 +99,22 @@ def build_app_dict(app, infos, folder): ), } +def reset(): + with open(CATALOG_LIST_PATH, "w") as f: + catalog_list = [{"id": "default", "url": "https://app.yunohost.org/default/"}] + yaml.safe_dump(catalog_list, f, default_flow_style=False) -if __name__ == "__main__": - build() + +def add(): + with open(CATALOG_LIST_PATH) as f: + catalog_list = yaml.load(f, Loader=yaml.FullLoader) + ids = [catalog["id"] for catalog in catalog_list] + if "custom" not in ids: + catalog_list.append({"id": "custom", "url": None}) + with open(CATALOG_LIST_PATH, "w") as f: + yaml.safe_dump(catalog_list, f, default_flow_style=False) + +def override(): + with open(CATALOG_LIST_PATH, "w") as f: + catalog_list = [{"id": "custom", "url": None}] + yaml.safe_dump(catalog_list, f, default_flow_style=False) diff --git a/ynh-dev b/ynh-dev index 9871bbd..dd6a184 100755 --- a/ynh-dev +++ b/ynh-dev @@ -19,7 +19,11 @@ function show_usage() { test [PKG] Deploy, update and run tests for some packages Tests for single modules and functions can ran with e.g. ./ynh-dev test yunohost/appurl:urlavailable - + catalog + build Rebuild the custom catalog + add Add the custom catalog in Yunohost catalog list + override Override default catalog with the custom catalog + reset Reset the catalog list to Yunohost's default EOF } @@ -489,7 +493,7 @@ function catalog() { assert_inside_vm local ACTION="$1" - local CUSTOM_APPS_FOLDER=${2:-"/ynh-dev/custom-apps"} + local CUSTOM_APPS_FOLDER=${2:-"/ynh-dev/custom-catalog"} local CUSTOM_CAT_PATH="${CUSTOM_APPS_FOLDER}/catalog.json" local CACHE_FOLDER="/var/cache/yunohost/repo" @@ -498,22 +502,22 @@ function catalog() case "${ACTION}" in build) info "Rebuilding custom app catalog" - python3 -c "from catalog_builder import build; build(folder='${CUSTOM_APPS_FOLDER}')" && success "Successfully build custom catalog list in '${CUSTOM_CAT_PATH}'" + python3 -c "from catalog_manager import build; build(folder='${CUSTOM_APPS_FOLDER}')" && success "Successfully build custom catalog list in '${CUSTOM_CAT_PATH}'" ;; add) info "Injecting custom catalog in YunoHost catalog list" create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json" - python3 -c "from catalog_list import add; add()" && success "Custom catalog '${CUSTOM_CAT_PATH}' added to catalog list" + python3 -c "from catalog_manager import add; add()" && success "Custom catalog '${CUSTOM_CAT_PATH}' added to catalog list" ;; override) info "Overriding default catalog by custom one" create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json" - python3 -c "from catalog_list import override; override()" && success "Default catalog is now overrided by '$CUSTOM_CAT_PATH'" + python3 -c "from catalog_manager import override; override()" && success "Default catalog is now overrided by '$CUSTOM_CAT_PATH'" ;; reset) info "Reseting to YunoHost default catalog list" [ -e "$CACHE_FOLDER/custom.json" ] && rm "$CACHE_FOLDER/custom.json" - python3 -c "from catalog_list import reset; reset()" || exit 1 + python3 -c "from catalog_manager import reset; reset()" || exit 1 success "Returned to default" ;; *) From 3221ad9f8756be6bbf09eae58f3bfbcf93149f62 Mon Sep 17 00:00:00 2001 From: axolotle Date: Sun, 20 Nov 2022 01:36:58 +0100 Subject: [PATCH 3/3] rename custom-catalog input to app.json.example --- custom-catalog/{apps.json => apps.json.example} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename custom-catalog/{apps.json => apps.json.example} (100%) diff --git a/custom-catalog/apps.json b/custom-catalog/apps.json.example similarity index 100% rename from custom-catalog/apps.json rename to custom-catalog/apps.json.example