From 206b430a9f66c698821f5fc56d1cfaa967e1361c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 1 Oct 2021 02:50:30 +0200 Subject: [PATCH] Offload appcatalog stuff to a separate file to limit the size of app.py --- .gitlab/ci/test.gitlab-ci.yml | 4 +- src/yunohost/app.py | 296 ++---------------- ...est_appscatalog.py => test_app_catalog.py} | 2 +- src/yunohost/tools.py | 4 +- 4 files changed, 31 insertions(+), 275 deletions(-) rename src/yunohost/tests/{test_appscatalog.py => test_app_catalog.py} (99%) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 6a422ea22..1aad46fbe 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -113,10 +113,10 @@ test-apps: test-appscatalog: extends: .test-stage script: - - python3 -m pytest src/yunohost/tests/test_appscatalog.py + - python3 -m pytest src/yunohost/tests/test_app_catalog.py only: changes: - - src/yunohost/app.py + - src/yunohost/app_calalog.py test-appurl: extends: .test-stage diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 011c953e0..7cddca3e9 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -31,15 +31,12 @@ import yaml import time import re import subprocess -import glob import tempfile from collections import OrderedDict -from typing import List +from typing import List, Tuple, Dict from moulinette import Moulinette, m18n -from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.network import download_json from moulinette.utils.process import run_commands, check_output from moulinette.utils.filesystem import ( read_file, @@ -48,8 +45,6 @@ from moulinette.utils.filesystem import ( read_yaml, write_to_file, write_to_json, - write_to_yaml, - mkdir, ) from yunohost.utils import packages @@ -64,17 +59,13 @@ from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory from yunohost.log import is_unit_operation, OperationLogger +from yunohost.app_catalog import app_catalog, app_search, _load_apps_catalog, app_fetchlist # noqa logger = getActionLogger("yunohost.app") APPS_SETTING_PATH = "/etc/yunohost/apps/" APP_TMP_WORKDIRS = "/var/cache/yunohost/app_tmp_work_dirs" -APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" -APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" -APPS_CATALOG_API_VERSION = 2 -APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" - re_app_instance_name = re.compile( r"^(?P[\w-]+?)(__(?P[1-9][0-9]*))?$" ) @@ -91,80 +82,6 @@ APP_FILES_TO_COPY = [ "doc", ] -def app_catalog(full=False, with_categories=False): - """ - Return a dict of apps available to installation from Yunohost's app catalog - """ - - # Get app list from catalog cache - catalog = _load_apps_catalog() - installed_apps = set(_installed_apps()) - - # Trim info for apps if not using --full - for app, infos in catalog["apps"].items(): - infos["installed"] = app in installed_apps - - infos["manifest"]["description"] = _value_for_locale( - infos["manifest"]["description"] - ) - - if not full: - catalog["apps"][app] = { - "description": infos["manifest"]["description"], - "level": infos["level"], - } - else: - infos["manifest"]["arguments"] = _set_default_ask_questions( - infos["manifest"].get("arguments", {}) - ) - - # Trim info for categories if not using --full - for category in catalog["categories"]: - category["title"] = _value_for_locale(category["title"]) - category["description"] = _value_for_locale(category["description"]) - for subtags in category.get("subtags", []): - subtags["title"] = _value_for_locale(subtags["title"]) - - if not full: - catalog["categories"] = [ - {"id": c["id"], "description": c["description"]} - for c in catalog["categories"] - ] - - if not with_categories: - return {"apps": catalog["apps"]} - else: - return {"apps": catalog["apps"], "categories": catalog["categories"]} - - -def app_search(string): - """ - Return a dict of apps whose description or name match the search string - """ - - # Retrieve a simple dict listing all apps - catalog_of_apps = app_catalog() - - # Selecting apps according to a match in app name or description - matching_apps = {"apps": {}} - for app in catalog_of_apps["apps"].items(): - if re.search(string, app[0], flags=re.IGNORECASE) or re.search( - string, app[1]["description"], flags=re.IGNORECASE - ): - matching_apps["apps"][app[0]] = app[1] - - return matching_apps - - -# Old legacy function... -def app_fetchlist(): - logger.warning( - "'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead" - ) - from yunohost.tools import tools_update - - tools_update(target="apps") - def app_list(full=False, installed=False, filter=None): """ @@ -1728,22 +1645,6 @@ ynh_app_config_run $1 return values -def _get_all_installed_apps_id(): - """ - Return something like: - ' * app1 - * app2 - * ...' - """ - - all_apps_ids = sorted(_installed_apps()) - - all_apps_ids_formatted = "\n * ".join(all_apps_ids) - all_apps_ids_formatted = "\n * " + all_apps_ids_formatted - - return all_apps_ids_formatted - - def _get_app_actions(app_id): "Get app config panel stored in json or in toml" actions_toml_path = os.path.join(APPS_SETTING_PATH, app_id, "actions.toml") @@ -2239,6 +2140,13 @@ def _extract_app_from_gitrepo(url: str, branch: str, revision: str, app_info={}: return manifest, extracted_app_folder +# +# ############################### # +# Small utilities # +# ############################### # +# + + def _is_installed(app: str) -> bool: """ Check if application is installed @@ -2264,6 +2172,22 @@ def _installed_apps() -> List[str]: return os.listdir(APPS_SETTING_PATH) +def _get_all_installed_apps_id(): + """ + Return something like: + ' * app1 + * app2 + * ...' + """ + + all_apps_ids = sorted(_installed_apps()) + + all_apps_ids_formatted = "\n * ".join(all_apps_ids) + all_apps_ids_formatted = "\n * " + all_apps_ids_formatted + + return all_apps_ids_formatted + + def _check_manifest_requirements(manifest: Dict, app: str): """Check if required packages are met from the manifest""" @@ -2476,176 +2400,6 @@ def _next_instance_number_for_app(app): i += 1 - -# -# ############################### # -# Applications list management # -# ############################### # -# - - -def _initialize_apps_catalog_system(): - """ - This function is meant to intialize the apps_catalog system with YunoHost's default app catalog. - """ - - default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}] - - try: - logger.debug( - "Initializing apps catalog system with YunoHost's default app list" - ) - write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list) - except Exception as e: - raise YunohostError( - "Could not initialize the apps catalog system... : %s" % str(e) - ) - - logger.success(m18n.n("apps_catalog_init_success")) - - -def _read_apps_catalog_list(): - """ - Read the json corresponding to the list of apps catalogs - """ - - try: - list_ = read_yaml(APPS_CATALOG_CONF) - # Support the case where file exists but is empty - # by returning [] if list_ is None - return list_ if list_ else [] - except Exception as e: - raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e)) - - -def _actual_apps_catalog_api_url(base_url): - - return "{base_url}/v{version}/apps.json".format( - base_url=base_url, version=APPS_CATALOG_API_VERSION - ) - - -def _update_apps_catalog(): - """ - Fetches the json for each apps_catalog and update the cache - - apps_catalog_list is for example : - [ {"id": "default", "url": "https://app.yunohost.org/default/"} ] - - Then for each apps_catalog, the actual json URL to be fetched is like : - https://app.yunohost.org/default/vX/apps.json - - And store it in : - /var/cache/yunohost/repo/default.json - """ - - apps_catalog_list = _read_apps_catalog_list() - - logger.info(m18n.n("apps_catalog_updating")) - - # Create cache folder if needed - if not os.path.exists(APPS_CATALOG_CACHE): - logger.debug("Initialize folder for apps catalog cache") - mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root") - - for apps_catalog in apps_catalog_list: - apps_catalog_id = apps_catalog["id"] - actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"]) - - # Fetch the json - try: - apps_catalog_content = download_json(actual_api_url) - except Exception as e: - raise YunohostError( - "apps_catalog_failed_to_download", - apps_catalog=apps_catalog_id, - error=str(e), - ) - - # Remember the apps_catalog api version for later - apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION - - # Save the apps_catalog data in the cache - cache_file = "{cache_folder}/{list}.json".format( - cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id - ) - try: - write_to_json(cache_file, apps_catalog_content) - except Exception as e: - raise YunohostError( - "Unable to write cache data for %s apps_catalog : %s" - % (apps_catalog_id, str(e)) - ) - - logger.success(m18n.n("apps_catalog_update_success")) - - -def _load_apps_catalog(): - """ - Read all the apps catalog cache files and build a single dict (merged_catalog) - corresponding to all known apps and categories - """ - - merged_catalog = {"apps": {}, "categories": []} - - for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]: - - # Let's load the json from cache for this catalog - cache_file = "{cache_folder}/{list}.json".format( - cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id - ) - - try: - apps_catalog_content = ( - read_json(cache_file) if os.path.exists(cache_file) else None - ) - except Exception as e: - raise YunohostError( - "Unable to read cache for apps_catalog %s : %s" % (cache_file, e), - raw_msg=True, - ) - - # Check that the version of the data matches version .... - # ... otherwise it means we updated yunohost in the meantime - # and need to update the cache for everything to be consistent - if ( - not apps_catalog_content - or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION - ): - logger.info(m18n.n("apps_catalog_obsolete_cache")) - _update_apps_catalog() - apps_catalog_content = read_json(cache_file) - - del apps_catalog_content["from_api_version"] - - # Add apps from this catalog to the output - for app, info in apps_catalog_content["apps"].items(): - - # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ... - # in which case we keep only the first one found) - if app in merged_catalog["apps"]: - logger.warning( - "Duplicate app %s found between apps catalog %s and %s" - % (app, apps_catalog_id, merged_catalog["apps"][app]["repository"]) - ) - continue - - info["repository"] = apps_catalog_id - merged_catalog["apps"][app] = info - - # Annnnd categories - merged_catalog["categories"] += apps_catalog_content["categories"] - - return merged_catalog - - -# -# ############################### # -# Small utilities # -# ############################### # -# - - def _make_tmp_workdir_for_app(app=None): # Create parent dir if it doesn't exists yet diff --git a/src/yunohost/tests/test_appscatalog.py b/src/yunohost/tests/test_app_catalog.py similarity index 99% rename from src/yunohost/tests/test_appscatalog.py rename to src/yunohost/tests/test_app_catalog.py index a2619a660..8423b868e 100644 --- a/src/yunohost/tests/test_appscatalog.py +++ b/src/yunohost/tests/test_app_catalog.py @@ -9,7 +9,7 @@ from moulinette import m18n from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml from yunohost.utils.error import YunohostError -from yunohost.app import ( +from yunohost.app_catalog import ( _initialize_apps_catalog_system, _read_apps_catalog_list, _update_apps_catalog, diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index ed8c04153..4cdd62d8f 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -37,10 +37,12 @@ from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_yaml, write_to_yaml from yunohost.app import ( - _update_apps_catalog, app_info, app_upgrade, +) +from yunohost.app_catalog import ( _initialize_apps_catalog_system, + _update_apps_catalog, ) from yunohost.domain import domain_add from yunohost.dyndns import _dyndns_available, _dyndns_provides