YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

Offload appcatalog stuff to a separate file to limit the size of app.py

Browse source
This commit is contained in:
Alexandre Aubin 2021-10-01 02:50:30 +02:00
parent f84d8618bc
commit 206b430a9f
4 changed files with 31 additions and 275 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitlab/ci/test.gitlab-ci.yml
View file

@ -113,10 +113,10 @@ test-apps:
test-appscatalog:
extends: .test-stage
script:
- python3 -m pytest src/yunohost/tests/test_appscatalog.py
- python3 -m pytest src/yunohost/tests/test_app_catalog.py
only:
changes:
- src/yunohost/app.py
- src/yunohost/app_calalog.py
test-appurl:
extends: .test-stage

296
src/yunohost/app.py
View file

@ -31,15 +31,12 @@ import yaml
import time
import re
import subprocess
import glob
import tempfile
from collections import OrderedDict
from typing import List
from typing import List, Tuple, Dict
from moulinette import Moulinette, m18n
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
from moulinette.utils.network import download_json
from moulinette.utils.process import run_commands, check_output
from moulinette.utils.filesystem import (
read_file,
@ -48,8 +45,6 @@ from moulinette.utils.filesystem import (
read_yaml,
write_to_file,
write_to_json,
write_to_yaml,
mkdir,
)
from yunohost.utils import packages
@ -64,17 +59,13 @@ from yunohost.utils.i18n import _value_for_locale
from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.utils.filesystem import free_space_in_directory
from yunohost.log import is_unit_operation, OperationLogger
from yunohost.app_catalog import app_catalog, app_search, _load_apps_catalog, app_fetchlist # noqa
logger = getActionLogger("yunohost.app")
APPS_SETTING_PATH = "/etc/yunohost/apps/"
APP_TMP_WORKDIRS = "/var/cache/yunohost/app_tmp_work_dirs"
APPS_CATALOG_CACHE = "/var/cache/yunohost/repo"
APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml"
APPS_CATALOG_API_VERSION = 2
APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
re_app_instance_name = re.compile(
r"^(?P<appid>[\w-]+?)(__(?P<appinstancenb>[1-9][0-9]*))?$"
)
@ -91,80 +82,6 @@ APP_FILES_TO_COPY = [
"doc",
]
def app_catalog(full=False, with_categories=False):
"""
Return a dict of apps available to installation from Yunohost's app catalog
"""
# Get app list from catalog cache
catalog = _load_apps_catalog()
installed_apps = set(_installed_apps())
# Trim info for apps if not using --full
for app, infos in catalog["apps"].items():
infos["installed"] = app in installed_apps
infos["manifest"]["description"] = _value_for_locale(
infos["manifest"]["description"]
)
if not full:
catalog["apps"][app] = {
"description": infos["manifest"]["description"],
"level": infos["level"],
}
else:
infos["manifest"]["arguments"] = _set_default_ask_questions(
infos["manifest"].get("arguments", {})
)
# Trim info for categories if not using --full
for category in catalog["categories"]:
category["title"] = _value_for_locale(category["title"])
category["description"] = _value_for_locale(category["description"])
for subtags in category.get("subtags", []):
subtags["title"] = _value_for_locale(subtags["title"])
if not full:
catalog["categories"] = [
{"id": c["id"], "description": c["description"]}
for c in catalog["categories"]
]
if not with_categories:
return {"apps": catalog["apps"]}
else:
return {"apps": catalog["apps"], "categories": catalog["categories"]}
def app_search(string):
"""
Return a dict of apps whose description or name match the search string
"""
# Retrieve a simple dict listing all apps
catalog_of_apps = app_catalog()
# Selecting apps according to a match in app name or description
matching_apps = {"apps": {}}
for app in catalog_of_apps["apps"].items():
if re.search(string, app[0], flags=re.IGNORECASE) or re.search(
string, app[1]["description"], flags=re.IGNORECASE
):
matching_apps["apps"][app[0]] = app[1]
return matching_apps
# Old legacy function...
def app_fetchlist():
logger.warning(
"'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead"
)
from yunohost.tools import tools_update
tools_update(target="apps")
def app_list(full=False, installed=False, filter=None):
"""
@ -1728,22 +1645,6 @@ ynh_app_config_run $1
return values
def _get_all_installed_apps_id():
"""
Return something like:
' * app1
* app2
* ...'
"""
all_apps_ids = sorted(_installed_apps())
all_apps_ids_formatted = "\n * ".join(all_apps_ids)
all_apps_ids_formatted = "\n * " + all_apps_ids_formatted
return all_apps_ids_formatted
def _get_app_actions(app_id):
"Get app config panel stored in json or in toml"
actions_toml_path = os.path.join(APPS_SETTING_PATH, app_id, "actions.toml")
@ -2239,6 +2140,13 @@ def _extract_app_from_gitrepo(url: str, branch: str, revision: str, app_info={}:
return manifest, extracted_app_folder
#
# ############################### #
# Small utilities #
# ############################### #
#
def _is_installed(app: str) -> bool:
"""
Check if application is installed
@ -2264,6 +2172,22 @@ def _installed_apps() -> List[str]:
return os.listdir(APPS_SETTING_PATH)
def _get_all_installed_apps_id():
"""
Return something like:
' * app1
* app2
* ...'
"""
all_apps_ids = sorted(_installed_apps())
all_apps_ids_formatted = "\n * ".join(all_apps_ids)
all_apps_ids_formatted = "\n * " + all_apps_ids_formatted
return all_apps_ids_formatted
def _check_manifest_requirements(manifest: Dict, app: str):
"""Check if required packages are met from the manifest"""
@ -2476,176 +2400,6 @@ def _next_instance_number_for_app(app):
i += 1
#
# ############################### #
# Applications list management #
# ############################### #
#
def _initialize_apps_catalog_system():
"""
This function is meant to intialize the apps_catalog system with YunoHost's default app catalog.
"""
default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}]
try:
logger.debug(
"Initializing apps catalog system with YunoHost's default app list"
)
write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list)
except Exception as e:
raise YunohostError(
"Could not initialize the apps catalog system... : %s" % str(e)
)
logger.success(m18n.n("apps_catalog_init_success"))
def _read_apps_catalog_list():
"""
Read the json corresponding to the list of apps catalogs
"""
try:
list_ = read_yaml(APPS_CATALOG_CONF)
# Support the case where file exists but is empty
# by returning [] if list_ is None
return list_ if list_ else []
except Exception as e:
raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e))
def _actual_apps_catalog_api_url(base_url):
return "{base_url}/v{version}/apps.json".format(
base_url=base_url, version=APPS_CATALOG_API_VERSION
)
def _update_apps_catalog():
"""
Fetches the json for each apps_catalog and update the cache
apps_catalog_list is for example :
[ {"id": "default", "url": "https://app.yunohost.org/default/"} ]
Then for each apps_catalog, the actual json URL to be fetched is like :
https://app.yunohost.org/default/vX/apps.json
And store it in :
/var/cache/yunohost/repo/default.json
"""
apps_catalog_list = _read_apps_catalog_list()
logger.info(m18n.n("apps_catalog_updating"))
# Create cache folder if needed
if not os.path.exists(APPS_CATALOG_CACHE):
logger.debug("Initialize folder for apps catalog cache")
mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root")
for apps_catalog in apps_catalog_list:
apps_catalog_id = apps_catalog["id"]
actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"])
# Fetch the json
try:
apps_catalog_content = download_json(actual_api_url)
except Exception as e:
raise YunohostError(
"apps_catalog_failed_to_download",
apps_catalog=apps_catalog_id,
error=str(e),
)
# Remember the apps_catalog api version for later
apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION
# Save the apps_catalog data in the cache
cache_file = "{cache_folder}/{list}.json".format(
cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
)
try:
write_to_json(cache_file, apps_catalog_content)
except Exception as e:
raise YunohostError(
"Unable to write cache data for %s apps_catalog : %s"
% (apps_catalog_id, str(e))
)
logger.success(m18n.n("apps_catalog_update_success"))
def _load_apps_catalog():
"""
Read all the apps catalog cache files and build a single dict (merged_catalog)
corresponding to all known apps and categories
"""
merged_catalog = {"apps": {}, "categories": []}
for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]:
# Let's load the json from cache for this catalog
cache_file = "{cache_folder}/{list}.json".format(
cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id
)
try:
apps_catalog_content = (
read_json(cache_file) if os.path.exists(cache_file) else None
)
except Exception as e:
raise YunohostError(
"Unable to read cache for apps_catalog %s : %s" % (cache_file, e),
raw_msg=True,
)
# Check that the version of the data matches version ....
# ... otherwise it means we updated yunohost in the meantime
# and need to update the cache for everything to be consistent
if (
not apps_catalog_content
or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION
):
logger.info(m18n.n("apps_catalog_obsolete_cache"))
_update_apps_catalog()
apps_catalog_content = read_json(cache_file)
del apps_catalog_content["from_api_version"]
# Add apps from this catalog to the output
for app, info in apps_catalog_content["apps"].items():
# (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ...
# in which case we keep only the first one found)
if app in merged_catalog["apps"]:
logger.warning(
"Duplicate app %s found between apps catalog %s and %s"
% (app, apps_catalog_id, merged_catalog["apps"][app]["repository"])
)
continue
info["repository"] = apps_catalog_id
merged_catalog["apps"][app] = info
# Annnnd categories
merged_catalog["categories"] += apps_catalog_content["categories"]
return merged_catalog
#
# ############################### #
# Small utilities #
# ############################### #
#
def _make_tmp_workdir_for_app(app=None):
# Create parent dir if it doesn't exists yet

2
src/yunohost/tests/test_appscatalog.py → src/yunohost/tests/test_app_catalog.py
View file

@ -9,7 +9,7 @@ from moulinette import m18n
from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml
from yunohost.utils.error import YunohostError
from yunohost.app import (
from yunohost.app_catalog import (
_initialize_apps_catalog_system,
_read_apps_catalog_list,
_update_apps_catalog,

4
src/yunohost/tools.py
View file

@ -37,10 +37,12 @@ from moulinette.utils.process import check_output, call_async_output
from moulinette.utils.filesystem import read_yaml, write_to_yaml
from yunohost.app import (
_update_apps_catalog,
app_info,
app_upgrade,
)
from yunohost.app_catalog import (
_initialize_apps_catalog_system,
_update_apps_catalog,
)
from yunohost.domain import domain_add
from yunohost.dyndns import _dyndns_available, _dyndns_provides