Offload appcatalog stuff to a separate file to limit the size of app.py

.gitlab/ci/test.gitlab-ci.yml

@@ -113,10 +113,10 @@ test-apps:
 test-appscatalog:
   extends: .test-stage
   script:
-    - python3 -m pytest src/yunohost/tests/test_appscatalog.py
+    - python3 -m pytest src/yunohost/tests/test_app_catalog.py
   only:
     changes:
-      - src/yunohost/app.py
+      - src/yunohost/app_calalog.py
 
 test-appurl:
   extends: .test-stage

src/yunohost/app.py

@@ -31,15 +31,12 @@ import yaml
 import time
 import re
 import subprocess
-import glob
 import tempfile
 from collections import OrderedDict
-from typing import List
+from typing import List, Tuple, Dict
 
 from moulinette import Moulinette, m18n
-from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.network import download_json
 from moulinette.utils.process import run_commands, check_output
 from moulinette.utils.filesystem import (
     read_file,
@@ -48,8 +45,6 @@ from moulinette.utils.filesystem import (
     read_yaml,
     write_to_file,
     write_to_json,
-    write_to_yaml,
-    mkdir,
 )
 
 from yunohost.utils import packages
@@ -64,17 +59,13 @@ from yunohost.utils.i18n import _value_for_locale
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.filesystem import free_space_in_directory
 from yunohost.log import is_unit_operation, OperationLogger
+from yunohost.app_catalog import app_catalog, app_search, _load_apps_catalog, app_fetchlist  # noqa
 
 logger = getActionLogger("yunohost.app")
 
 APPS_SETTING_PATH = "/etc/yunohost/apps/"
 APP_TMP_WORKDIRS = "/var/cache/yunohost/app_tmp_work_dirs"
 
-APPS_CATALOG_CACHE = "/var/cache/yunohost/repo"
-APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml"
-APPS_CATALOG_API_VERSION = 2
-APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
-
 re_app_instance_name = re.compile(
     r"^(?P<appid>[\w-]+?)(__(?P<appinstancenb>[1-9][0-9]*))?$"
 )
@@ -91,80 +82,6 @@ APP_FILES_TO_COPY = [
     "doc",
 ]
 
-def app_catalog(full=False, with_categories=False):
-    """
-    Return a dict of apps available to installation from Yunohost's app catalog
-    """
-
-    # Get app list from catalog cache
-    catalog = _load_apps_catalog()
-    installed_apps = set(_installed_apps())
-
-    # Trim info for apps if not using --full
-    for app, infos in catalog["apps"].items():
-        infos["installed"] = app in installed_apps
-
-        infos["manifest"]["description"] = _value_for_locale(
-            infos["manifest"]["description"]
-        )
-
-        if not full:
-            catalog["apps"][app] = {
-                "description": infos["manifest"]["description"],
-                "level": infos["level"],
-            }
-        else:
-            infos["manifest"]["arguments"] = _set_default_ask_questions(
-                infos["manifest"].get("arguments", {})
-            )
-
-    # Trim info for categories if not using --full
-    for category in catalog["categories"]:
-        category["title"] = _value_for_locale(category["title"])
-        category["description"] = _value_for_locale(category["description"])
-        for subtags in category.get("subtags", []):
-            subtags["title"] = _value_for_locale(subtags["title"])
-
-    if not full:
-        catalog["categories"] = [
-            {"id": c["id"], "description": c["description"]}
-            for c in catalog["categories"]
-        ]
-
-    if not with_categories:
-        return {"apps": catalog["apps"]}
-    else:
-        return {"apps": catalog["apps"], "categories": catalog["categories"]}
-
-
-def app_search(string):
-    """
-    Return a dict of apps whose description or name match the search string
-    """
-
-    # Retrieve a simple dict listing all apps
-    catalog_of_apps = app_catalog()
-
-    # Selecting apps according to a match in app name or description
-    matching_apps = {"apps": {}}
-    for app in catalog_of_apps["apps"].items():
-        if re.search(string, app[0], flags=re.IGNORECASE) or re.search(
-            string, app[1]["description"], flags=re.IGNORECASE
-        ):
-            matching_apps["apps"][app[0]] = app[1]
-
-    return matching_apps
-
-
-# Old legacy function...
-def app_fetchlist():
-    logger.warning(
-        "'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead"
-    )
-    from yunohost.tools import tools_update
-
-    tools_update(target="apps")
-
 
 def app_list(full=False, installed=False, filter=None):
     """
@@ -1728,22 +1645,6 @@ ynh_app_config_run $1
         return values
 
 
-def _get_all_installed_apps_id():
-    """
-    Return something like:
-       ' * app1
-         * app2
-         * ...'
-    """
-
-    all_apps_ids = sorted(_installed_apps())
-
-    all_apps_ids_formatted = "\n * ".join(all_apps_ids)
-    all_apps_ids_formatted = "\n * " + all_apps_ids_formatted
-
-    return all_apps_ids_formatted
-
-
 def _get_app_actions(app_id):
     "Get app config panel stored in json or in toml"
     actions_toml_path = os.path.join(APPS_SETTING_PATH, app_id, "actions.toml")
@@ -2239,6 +2140,13 @@ def _extract_app_from_gitrepo(url: str, branch: str, revision: str, app_info={}:
     return manifest, extracted_app_folder
 
 
+#
+# ############################### #
+#        Small utilities          #
+# ############################### #
+#
+
+
 def _is_installed(app: str) -> bool:
     """
     Check if application is installed
@@ -2264,6 +2172,22 @@ def _installed_apps() -> List[str]:
     return os.listdir(APPS_SETTING_PATH)
 
 
+def _get_all_installed_apps_id():
+    """
+    Return something like:
+       ' * app1
+         * app2
+         * ...'
+    """
+
+    all_apps_ids = sorted(_installed_apps())
+
+    all_apps_ids_formatted = "\n * ".join(all_apps_ids)
+    all_apps_ids_formatted = "\n * " + all_apps_ids_formatted
+
+    return all_apps_ids_formatted
+
+
 def _check_manifest_requirements(manifest: Dict, app: str):
     """Check if required packages are met from the manifest"""
 
@@ -2476,176 +2400,6 @@ def _next_instance_number_for_app(app):
             i += 1
 
 
-
-#
-# ############################### #
-#  Applications list management   #
-# ############################### #
-#
-
-
-def _initialize_apps_catalog_system():
-    """
-    This function is meant to intialize the apps_catalog system with YunoHost's default app catalog.
-    """
-
-    default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}]
-
-    try:
-        logger.debug(
-            "Initializing apps catalog system with YunoHost's default app list"
-        )
-        write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list)
-    except Exception as e:
-        raise YunohostError(
-            "Could not initialize the apps catalog system... : %s" % str(e)
-        )
-
-    logger.success(m18n.n("apps_catalog_init_success"))
-
-
-def _read_apps_catalog_list():
-    """
-    Read the json corresponding to the list of apps catalogs
-    """
-
-    try:
-        list_ = read_yaml(APPS_CATALOG_CONF)
-        # Support the case where file exists but is empty
-        # by returning [] if list_ is None
-        return list_ if list_ else []
-    except Exception as e:
-        raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e))
-
-
-def _actual_apps_catalog_api_url(base_url):
-
-    return "{base_url}/v{version}/apps.json".format(
-        base_url=base_url, version=APPS_CATALOG_API_VERSION
-    )
-
-
-def _update_apps_catalog():
-    """
-    Fetches the json for each apps_catalog and update the cache
-
-    apps_catalog_list is for example :
-     [   {"id": "default", "url": "https://app.yunohost.org/default/"}  ]
-
-    Then for each apps_catalog, the actual json URL to be fetched is like :
-       https://app.yunohost.org/default/vX/apps.json
-
-    And store it in :
-        /var/cache/yunohost/repo/default.json
-    """
-
-    apps_catalog_list = _read_apps_catalog_list()
-
-    logger.info(m18n.n("apps_catalog_updating"))
-
-    # Create cache folder if needed
-    if not os.path.exists(APPS_CATALOG_CACHE):
-        logger.debug("Initialize folder for apps catalog cache")
-        mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root")
-
-    for apps_catalog in apps_catalog_list:
-        apps_catalog_id = apps_catalog["id"]
-        actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"])
-
-        # Fetch the json
-        try:
-            apps_catalog_content = download_json(actual_api_url)
-        except Exception as e:
-            raise YunohostError(
-                "apps_catalog_failed_to_download",
-                apps_catalog=apps_catalog_id,
-                error=str(e),
-            )
-
-        # Remember the apps_catalog api version for later
-        apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION
-
-        # Save the apps_catalog data in the cache
-        cache_file = "{cache_folder}/{list}.json".format(
-            cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
-        )
-        try:
-            write_to_json(cache_file, apps_catalog_content)
-        except Exception as e:
-            raise YunohostError(
-                "Unable to write cache data for %s apps_catalog : %s"
-                % (apps_catalog_id, str(e))
-            )
-
-    logger.success(m18n.n("apps_catalog_update_success"))
-
-
-def _load_apps_catalog():
-    """
-    Read all the apps catalog cache files and build a single dict (merged_catalog)
-    corresponding to all known apps and categories
-    """
-
-    merged_catalog = {"apps": {}, "categories": []}
-
-    for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]:
-
-        # Let's load the json from cache for this catalog
-        cache_file = "{cache_folder}/{list}.json".format(
-            cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
-        )
-
-        try:
-            apps_catalog_content = (
-                read_json(cache_file) if os.path.exists(cache_file) else None
-            )
-        except Exception as e:
-            raise YunohostError(
-                "Unable to read cache for apps_catalog %s : %s" % (cache_file, e),
-                raw_msg=True,
-            )
-
-        # Check that the version of the data matches version ....
-        # ... otherwise it means we updated yunohost in the meantime
-        # and need to update the cache for everything to be consistent
-        if (
-            not apps_catalog_content
-            or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION
-        ):
-            logger.info(m18n.n("apps_catalog_obsolete_cache"))
-            _update_apps_catalog()
-            apps_catalog_content = read_json(cache_file)
-
-        del apps_catalog_content["from_api_version"]
-
-        # Add apps from this catalog to the output
-        for app, info in apps_catalog_content["apps"].items():
-
-            # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ...
-            #         in which case we keep only the first one found)
-            if app in merged_catalog["apps"]:
-                logger.warning(
-                    "Duplicate app %s found between apps catalog %s and %s"
-                    % (app, apps_catalog_id, merged_catalog["apps"][app]["repository"])
-                )
-                continue
-
-            info["repository"] = apps_catalog_id
-            merged_catalog["apps"][app] = info
-
-        # Annnnd categories
-        merged_catalog["categories"] += apps_catalog_content["categories"]
-
-    return merged_catalog
-
-
-#
-# ############################### #
-#        Small utilities          #
-# ############################### #
-#
-
-
 def _make_tmp_workdir_for_app(app=None):
 
     # Create parent dir if it doesn't exists yet

src/yunohost/tests/test_app_catalog.py

@@ -9,7 +9,7 @@ from moulinette import m18n
 from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml
 
 from yunohost.utils.error import YunohostError
-from yunohost.app import (
+from yunohost.app_catalog import (
     _initialize_apps_catalog_system,
     _read_apps_catalog_list,
     _update_apps_catalog,

src/yunohost/tools.py

@@ -37,10 +37,12 @@ from moulinette.utils.process import check_output, call_async_output
 from moulinette.utils.filesystem import read_yaml, write_to_yaml
 
 from yunohost.app import (
-    _update_apps_catalog,
     app_info,
     app_upgrade,
+)
+from yunohost.app_catalog import (
     _initialize_apps_catalog_system,
+    _update_apps_catalog,
 )
 from yunohost.domain import domain_add
 from yunohost.dyndns import _dyndns_available, _dyndns_provides