From 54a7f7a570ea93c495b2bbf1f924ed174280b5c9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 15 Aug 2021 01:41:06 +0200 Subject: [PATCH 01/54] manifestv2: auto-save install settings, auto-load all settings --- src/yunohost/app.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index fe5281384..c2adbd550 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -831,6 +831,18 @@ def app_install( "install_time": int(time.time()), "current_revision": manifest.get("remote", {}).get("revision", "?"), } + + # If packaging_format v2+, save all install questions as settings + packaging_format = int(manifest.get("packaging_format", 0)) + if packaging_format >= 2: + for arg_name, arg_value_and_type in args_odict.items(): + + # ... except is_public because it should not be saved and should only be about initializing permisisons + if arg_name == "is_public": + continue + + app_settings[arg_name] = arg_value_and_type[0] + _set_app_settings(app_instance_name, app_settings) # Move scripts and manifest to the right place @@ -2401,6 +2413,24 @@ def _make_environment_for_app_script( for arg_name, arg_value in args.items(): env_dict["YNH_%s%s" % (args_prefix, arg_name.upper())] = str(arg_value) + # If packaging format v2, load all settings + packaging_format = int(manifest.get("packaging_format", 0)) + if packaging_format >= 2: + env_dict["app"] = app + for setting_name, setting_value in _get_app_settings(app): + + # Ignore special internal settings like checksum__ + # (not a huge deal to load them but idk...) + if setting_name.startswith("checksum__"): + continue + + env_dict[setting_name] = str(setting_value) + + # Special weird case for backward compatibility... + # 'path' was loaded into 'path_url' ..... + if 'path' in env_dict: + env_dict["path_url"] = env_dict["path"] + return env_dict From 953eb0cc1d20fb2a711c24591929465d79ea7e6e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 15 Aug 2021 02:54:58 +0200 Subject: [PATCH 02/54] manifestv2: on-the-fly convert v1 manifest to v2 --- src/yunohost/app.py | 90 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 16 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index c2adbd550..64766f96c 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -32,6 +32,7 @@ import time import re import subprocess import tempfile +import copy from collections import OrderedDict from typing import List, Tuple, Dict, Any @@ -1929,21 +1930,7 @@ def _get_manifest_of_app(path): # ¦ ¦ }, if os.path.exists(os.path.join(path, "manifest.toml")): - manifest_toml = read_toml(os.path.join(path, "manifest.toml")) - - manifest = manifest_toml.copy() - - install_arguments = [] - for name, values in ( - manifest_toml.get("arguments", {}).get("install", {}).items() - ): - args = values.copy() - args["name"] = name - - install_arguments.append(args) - - manifest["arguments"]["install"] = install_arguments - + manifest = read_toml(os.path.join(path, "manifest.toml")) elif os.path.exists(os.path.join(path, "manifest.json")): manifest = read_json(os.path.join(path, "manifest.json")) else: @@ -1953,7 +1940,78 @@ def _get_manifest_of_app(path): raw_msg=True, ) - manifest["arguments"] = _set_default_ask_questions(manifest.get("arguments", {})) + if int(manifest.get("packaging_format", 0)) <= 1: + manifest = _convert_v1_manifest_to_v2(manifest) + + manifest["install"] = _set_default_ask_questions(manifest.get("install", {})) + return manifest + + +def _convert_v1_manifest_to_v2(manifest): + + manifest = copy.deepcopy(manifest) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost"), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + manifest["resources"] = { + "disk": { + "build": "50M", # This is an *estimate* minimum value for the disk needed at build time (e.g. during install/upgrade) and during regular usage + "usage": "50M" # Please only use round values such as: 10M, 100M, 200M, 400M, 800M, 1G, 2G, 4G, 8G + }, + "ram": { + "build": "50M", # This is an *estimate* minimum value for the RAM needed at build time (i.e. during install/upgrade) and during regular usage + "usage": "10M", # Please only use round values like ["10M", "100M", "200M", "400M", "800M", "1G", "2G", "4G", "8G"] + "include_swap": False + }, + "route": {}, + "install_dir": { + "base_dir": "/var/www/", # This means that the app shall be installed in /var/www/$app which is the standard for webapps. You may change this to /opt/ if the app is a system app. + "alias": "final_path" + } + } + + if "domain" in manifest["install"] and "path" in manifest["install"]: + manifest["resources"]["route"]["url"] = "{domain}{path}" + elif "path" not in manifest["install"]: + manifest["resources"]["route"]["url"] = "{domain}/" + else: + del manifest["resources"]["route"] + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + return manifest From 56ac85af37a4161d947f8d4bb3d9432fff70e394 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 15 Aug 2021 02:55:20 +0200 Subject: [PATCH 03/54] Misc tweaks to handle new install questions format --- src/yunohost/app.py | 78 ++++++++++++++----------------------- src/yunohost/app_catalog.py | 4 +- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 64766f96c..35ad2be36 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -95,7 +95,6 @@ APP_FILES_TO_COPY = [ "doc", ] - def app_list(full=False, installed=False, filter=None): """ List installed apps @@ -161,8 +160,8 @@ def app_info(app, full=False): ret["setting_path"] = setting_path ret["manifest"] = local_manifest - ret["manifest"]["arguments"] = _set_default_ask_questions( - ret["manifest"].get("arguments", {}) + ret["manifest"]["install"] = _set_default_ask_questions( + ret["manifest"].get("install", {}) ) ret["settings"] = settings @@ -179,7 +178,7 @@ def app_info(app, full=False): os.path.join(setting_path, "scripts", "backup") ) and os.path.exists(os.path.join(setting_path, "scripts", "restore")) ret["supports_multi_instance"] = is_true( - local_manifest.get("multi_instance", False) + local_manifest.get("integration", {}).get("multi_instance", False) ) ret["supports_config_panel"] = os.path.exists( os.path.join(setting_path, "config_panel.toml") @@ -836,13 +835,13 @@ def app_install( # If packaging_format v2+, save all install questions as settings packaging_format = int(manifest.get("packaging_format", 0)) if packaging_format >= 2: - for arg_name, arg_value_and_type in args_odict.items(): + for arg_name, arg_value in args.items(): - # ... except is_public because it should not be saved and should only be about initializing permisisons + # ... except is_public .... if arg_name == "is_public": continue - app_settings[arg_name] = arg_value_and_type[0] + app_settings[arg_name] = arg_value _set_app_settings(app_instance_name, app_settings) @@ -1751,15 +1750,7 @@ def _get_app_actions(app_id): for key, value in toml_actions.items(): action = dict(**value) action["id"] = key - - arguments = [] - for argument_name, argument in value.get("arguments", {}).items(): - argument = dict(**argument) - argument["name"] = argument_name - - arguments.append(argument) - - action["arguments"] = arguments + action["arguments"] = value.get("arguments", {}) actions.append(action) return actions @@ -2015,21 +2006,19 @@ def _convert_v1_manifest_to_v2(manifest): return manifest -def _set_default_ask_questions(arguments): +def _set_default_ask_questions(questions, script_name="install"): # arguments is something like - # { "install": [ - # { "name": "domain", + # { "domain": + # { # "type": "domain", # .... # }, - # { "name": "path", - # "type": "path" + # "path": { + # "type": "path", # ... # }, # ... - # ], - # "upgrade": [ ... ] # } # We set a default for any question with these matching (type, name) @@ -2041,38 +2030,29 @@ def _set_default_ask_questions(arguments): ("path", "path"), # i18n: app_manifest_install_ask_path ("password", "password"), # i18n: app_manifest_install_ask_password ("user", "admin"), # i18n: app_manifest_install_ask_admin - ("boolean", "is_public"), - ] # i18n: app_manifest_install_ask_is_public + ("boolean", "is_public"), # i18n: app_manifest_install_ask_is_public + ] - for script_name, arg_list in arguments.items(): + for question_name, question in questions.items(): + question["name"] = question_name - # We only support questions for install so far, and for other - if script_name != "install": - continue - - for arg in arg_list: - - # Do not override 'ask' field if provided by app ?... Or shall we ? - # if "ask" in arg: - # continue - - # If this arg corresponds to a question with default ask message... - if any( - (arg.get("type"), arg["name"]) == question - for question in questions_with_default - ): - # The key is for example "app_manifest_install_ask_domain" - key = "app_manifest_%s_ask_%s" % (script_name, arg["name"]) - arg["ask"] = m18n.n(key) + # If this question corresponds to a question with default ask message... + if any( + (question.get("type"), question["name"]) == q + for q in questions_with_default + ): + # The key is for example "app_manifest_install_ask_domain" + key = "app_manifest_%s_ask_%s" % (script_name, question["name"]) + question["ask"] = m18n.n(key) # Also it in fact doesn't make sense for any of those questions to have an example value nor a default value... - if arg.get("type") in ["domain", "user", "password"]: + if question.get("type") in ["domain", "user", "password"]: if "example" in arg: - del arg["example"] + del question["example"] if "default" in arg: - del arg["domain"] + del question["domain"] - return arguments + return questions def _is_app_repo_url(string: str) -> bool: @@ -2306,7 +2286,7 @@ def _check_manifest_requirements(manifest: Dict): """Check if required packages are met from the manifest""" packaging_format = int(manifest.get("packaging_format", 0)) - if packaging_format not in [0, 1]: + if packaging_format not in [2]: raise YunohostValidationError("app_packaging_format_not_supported") requirements = manifest.get("requirements", dict()) diff --git a/src/yunohost/app_catalog.py b/src/yunohost/app_catalog.py index e4ffa1db6..575f0bf1b 100644 --- a/src/yunohost/app_catalog.py +++ b/src/yunohost/app_catalog.py @@ -58,8 +58,8 @@ def app_catalog(full=False, with_categories=False): "level": infos["level"], } else: - infos["manifest"]["arguments"] = _set_default_ask_questions( - infos["manifest"].get("arguments", {}) + infos["manifest"]["install"] = _set_default_ask_questions( + infos["manifest"].get("install", {}) ) # Trim info for categories if not using --full From fe6ebe8e66741d827398097526445219b315c4ce Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 Oct 2021 09:34:28 +0200 Subject: [PATCH 04/54] Always define YNH_ACTION --- src/yunohost/app.py | 22 ++++++++++++++-------- src/yunohost/backup.py | 4 ++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 35ad2be36..c86c88747 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -408,7 +408,7 @@ def app_change_url(operation_logger, app, domain, path): tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app) # Prepare env. var. to pass to script - env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app) + env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app, action="change_url") env_dict["YNH_APP_OLD_DOMAIN"] = old_domain env_dict["YNH_APP_OLD_PATH"] = old_path env_dict["YNH_APP_NEW_DOMAIN"] = domain @@ -562,7 +562,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( - app_instance_name, workdir=extracted_app_folder + app_instance_name, workdir=extracted_app_folder, action="upgrade" ) env_dict["YNH_APP_UPGRADE_TYPE"] = upgrade_type env_dict["YNH_APP_MANIFEST_VERSION"] = str(app_new_version) @@ -868,7 +868,7 @@ def app_install( # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( - app_instance_name, args=args, workdir=extracted_app_folder + app_instance_name, args=args, workdir=extracted_app_folder, action="install" ) env_dict_for_logging = env_dict.copy() @@ -931,7 +931,7 @@ def app_install( # Setup environment for remove script env_dict_remove = _make_environment_for_app_script( - app_instance_name, workdir=extracted_app_folder + app_instance_name, workdir=extracted_app_folder, action="remove" ) # Execute remove script @@ -1046,7 +1046,7 @@ def app_remove(operation_logger, app, purge=False): env_dict = {} app_id, app_instance_nb = _parse_app_instance_name(app) - env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app) + env_dict = _make_environment_for_app_script(app, workdir=tmp_workdir_for_app, action="remove") env_dict["YNH_APP_PURGE"] = str(1 if purge else 0) operation_logger.extra.update({"env": env_dict}) @@ -1541,9 +1541,8 @@ def app_action_run(operation_logger, app, action, args=None): tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app) env_dict = _make_environment_for_app_script( - app, args=args, args_prefix="ACTION_", workdir=tmp_workdir_for_app + app, args=args, args_prefix="ACTION_", workdir=tmp_workdir_for_app, action=action ) - env_dict["YNH_ACTION"] = action _, action_script = tempfile.mkstemp(dir=tmp_workdir_for_app) @@ -2430,7 +2429,11 @@ def _assert_no_conflicting_apps(domain, path, ignore_app=None, full_domain=False def _make_environment_for_app_script( - app, args={}, args_prefix="APP_ARG_", workdir=None + app, + args={}, + args_prefix="APP_ARG_", + workdir=None, + action=None ): app_setting_path = os.path.join(APPS_SETTING_PATH, app) @@ -2448,6 +2451,9 @@ def _make_environment_for_app_script( if workdir: env_dict["YNH_APP_BASEDIR"] = workdir + if action: + env_dict["YNH_ACTION"] = action + for arg_name, arg_value in args.items(): env_dict["YNH_%s%s" % (args_prefix, arg_name.upper())] = str(arg_value) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index cce66597a..1b6aebd79 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -577,7 +577,7 @@ class BackupManager: env_var["YNH_BACKUP_CSV"] = tmp_csv if app is not None: - env_var.update(_make_environment_for_app_script(app)) + env_var.update(_make_environment_for_app_script(app, action="backup")) env_var["YNH_APP_BACKUP_DIR"] = os.path.join( self.work_dir, "apps", app, "backup" ) @@ -1490,7 +1490,7 @@ class RestoreManager: # FIXME : workdir should be a tmp workdir app_workdir = os.path.join(self.work_dir, "apps", app_instance_name, "settings") env_dict = _make_environment_for_app_script( - app_instance_name, workdir=app_workdir + app_instance_name, workdir=app_workdir, action="restore" ) env_dict.update( { From fe6d9c2617d1e947d4f4946049cfa663a6f97042 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 13 Oct 2021 22:56:51 +0200 Subject: [PATCH 05/54] manifestv2: automatically trigger ynh_abort_if_errors (set -eu) for scripts except remove --- data/helpers.d/utils | 6 ++++++ src/yunohost/app.py | 15 ++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 453a1ab94..57d5de56a 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -61,6 +61,12 @@ ynh_abort_if_errors() { trap ynh_exit_properly EXIT # Capturing exit signals on shell script } +# When running an app script with packaging format >= 2, auto-enable ynh_abort_if_errors except for remove script +if [[ ${YNH_APP_PACKAGING_FORMAT:-0} -ge 2 ]] && [[ ${YNH_APP_ACTION} != "remove" ]] +then + ynh_abort_if_errors +fi + # Download, check integrity, uncompress and patch the source from app.src # # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a0c6d55d1..7488a1646 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1371,7 +1371,7 @@ def app_register_url(app, domain, path): raise YunohostValidationError("app_already_installed_cant_change_url") # Check the url is available - _assert_no_conflicting_apps(domain, path) + _assert_no_conflicting_apps(domain, path, ignore_app=app) app_setting(app, "domain", value=domain) app_setting(app, "path", value=path) @@ -1930,7 +1930,9 @@ def _get_manifest_of_app(path): raw_msg=True, ) - if int(manifest.get("packaging_format", 0)) <= 1: + manifest["packaging_format"] = float(str(manifest.get("packaging_format", "")).strip() or "0") + + if manifest["packaging_format"] < 2: manifest = _convert_v1_manifest_to_v2(manifest) manifest["install"] = _set_default_ask_questions(manifest.get("install", {})) @@ -2284,8 +2286,7 @@ def _get_all_installed_apps_id(): def _check_manifest_requirements(manifest: Dict): """Check if required packages are met from the manifest""" - packaging_format = int(manifest.get("packaging_format", 0)) - if packaging_format not in [2]: + if manifest["packaging_format"] not in [1, 2]: raise YunohostValidationError("app_packaging_format_not_supported") requirements = manifest.get("requirements", dict()) @@ -2446,20 +2447,20 @@ def _make_environment_for_app_script( "YNH_APP_INSTANCE_NAME": app, "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), "YNH_APP_MANIFEST_VERSION": manifest.get("version", "?"), + "YNH_APP_PACKAGING_FORMAT": str(manifest["packaging_format"]), } if workdir: env_dict["YNH_APP_BASEDIR"] = workdir if action: - env_dict["YNH_ACTION"] = action + env_dict["YNH_APP_ACTION"] = action for arg_name, arg_value in args.items(): env_dict["YNH_%s%s" % (args_prefix, arg_name.upper())] = str(arg_value) # If packaging format v2, load all settings - packaging_format = int(manifest.get("packaging_format", 0)) - if packaging_format >= 2: + if manifest["packaging_format"] >= 2: env_dict["app"] = app for setting_name, setting_value in _get_app_settings(app): From 859b3b6f128edafec363a06d41a4fe2042dcf665 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 13 Oct 2021 22:57:17 +0200 Subject: [PATCH 06/54] Yoloimplement some resources classes, not used for anything for now --- src/yunohost/utils/resources.py | 247 ++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 src/yunohost/utils/resources.py diff --git a/src/yunohost/utils/resources.py b/src/yunohost/utils/resources.py new file mode 100644 index 000000000..42ae94d83 --- /dev/null +++ b/src/yunohost/utils/resources.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2021 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" +import os +import copy +import psutil +from typing import Dict, Any + +from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.utils.filesystem import free_space_in_directory + + +class AppResource: + + def __init__(self, properties: Dict[str, Any], app_id: str, app_settings): + + for key, value in self.default_properties.items(): + setattr(self, key, value) + + for key, value in properties: + setattr(self. key, value) + + +M = 1024 ** 2 +G = 1024 * M +sizes = { + "10M": 10 * M, + "20M": 20 * M, + "40M": 40 * M, + "80M": 80 * M, + "100M": 100 * M, + "200M": 200 * M, + "400M": 400 * M, + "800M": 800 * M, + "1G": 1 * G, + "2G": 2 * G, + "4G": 4 * G, + "8G": 8 * G, + "10G": 10 * G, + "20G": 20 * G, + "40G": 40 * G, + "80G": 80 * G, +} + + +class DiskAppResource(AppResource): + type = "disk" + + default_properties = { + "space": "10M", + } + + def __init__(self, *args, **kwargs): + super().__init__(self, *args, **kwargs) + # FIXME: better error handling + assert self.space in sizes + + def provision_or_update(self, context: Dict): + + if free_space_in_directory("/") <= sizes[self.space] \ + or free_space_in_directory("/var") <= sizes[self.space]: + raise YunohostValidationError("Not enough disk space") # FIXME: i18n / better messaging + + def deprovision(self, context: Dict): + pass + + +class RamAppResource(AppResource): + type = "ram" + + default_properties = { + "build": "10M", + "runtime": "10M", + "include_swap": False + } + + def __init__(self, *args, **kwargs): + super().__init__(self, *args, **kwargs) + # FIXME: better error handling + assert self.build in sizes + assert self.runtime in sizes + assert isinstance(self.include_swap, bool) + + def provision_or_update(self, context: Dict): + + memory = psutil.virtual_memory().available + if self.include_swap: + memory += psutil.swap_memory().available + + max_size = max(sizes[self.build], sizes[self.runtime]) + + if memory <= max_size: + raise YunohostValidationError("Not enough RAM/swap") # FIXME: i18n / better messaging + + def deprovision(self, context: Dict): + pass + + +class WebpathAppResource(AppResource): + type = "webpath" + + default_properties = { + "url": "__DOMAIN____PATH__" + } + + def provision_or_update(self, context: Dict): + + from yunohost.app import _assert_no_conflicting_apps + + # Check the url is available + domain = context["app_settings"]["domain"] + path = context["app_settings"]["path"] or "/" + _assert_no_conflicting_apps(domain, path, ignore_app=context["app"]) + context["app_settings"]["path"] = path + + if context["app_action"] == "install": + # Initially, the .main permission is created with no url at all associated + # When the app register/books its web url, we also add the url '/' + # (meaning the root of the app, domain.tld/path/) + # and enable the tile to the SSO, and both of this should match 95% of apps + # For more specific cases, the app is free to change / add urls or disable + # the tile using the permission helpers. + permission_url(app + ".main", url="/", sync_perm=False) + user_permission_update(app + ".main", show_tile=True, sync_perm=False) + permission_sync_to_user() + + def deprovision(self, context: Dict): + del context["app_settings"]["domain"] + del context["app_settings"]["path"] + + +class PortAppResource(AppResource): + type = "port" + + default_properties = { + "value": 1000 + } + + def _port_is_used(self, port): + + # FIXME : this could be less brutal than two os.system ... + cmd1 = "ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ':%s$'" % port + # This second command is mean to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up) + cmd2 = f"grep -q \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml" + return os.system(cmd1) == 0 and os.system(cmd2) == 0 + + def provision_or_update(self, context: str): + + # Don't do anything if port already defined ? + if context["app_settings"].get("port"): + return + + port = self.value + while self._port_is_used(port): + port += 1 + + context["app_settings"]["port"] = port + + def deprovision(self, context: Dict): + raise NotImplementedError() + + +class UserAppResource(AppResource): + type = "user" + + default_properties = { + "username": "__APP__", + "home_dir": "/var/www/__APP__", + "use_shell": False, + "groups": [] + } + + def provision_or_update(self, context: str): + raise NotImplementedError() + + def deprovision(self, context: Dict): + raise NotImplementedError() + + +class InstalldirAppResource(AppResource): + type = "installdir" + + default_properties = { + "dir": "/var/www/__APP__", + "alias": "final_path" + } + + def provision_or_update(self, context: Dict): + + if context["app_action"] in ["install", "restore"]: + if os.path.exists(self.dir): + raise YunohostValidationError(f"Path {self.dir} already exists") + + if "installdir" not in context["app_settings"]: + context["app_settings"]["installdir"] = self.dir + context["app_settings"][self.alias] = context["app_settings"]["installdir"] + + def deprovision(self, context: Dict): + # FIXME: should it rm the directory during remove/deprovision ? + pass + + +class DatadirAppResource(AppResource): + type = "datadir" + + default_properties = { + "dir": "/home/yunohost.app/__APP__", + } + + def provision_or_update(self, context: Dict): + if "datadir" not in context["app_settings"]: + context["app_settings"]["datadir"] = self.dir + + def deprovision(self, context: Dict): + # FIXME: should it rm the directory during remove/deprovision ? + pass + + +class DBAppResource(AppResource): + type = "db" + + default_properties = { + "type": "mysql" + } + + def provision_or_update(self, context: str): + raise NotImplementedError() + + def deprovision(self, context: Dict): + raise NotImplementedError() From 3084359155bf29c3cb69cc654cac0d19876e109f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 1 Nov 2021 22:40:52 +0100 Subject: [PATCH 07/54] Typos --- data/helpers.d/utils | 2 +- src/yunohost/app.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 57d5de56a..02fd10ba9 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -62,7 +62,7 @@ ynh_abort_if_errors() { } # When running an app script with packaging format >= 2, auto-enable ynh_abort_if_errors except for remove script -if [[ ${YNH_APP_PACKAGING_FORMAT:-0} -ge 2 ]] && [[ ${YNH_APP_ACTION} != "remove" ]] +if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 && [[ ${YNH_APP_ACTION} != "remove" ]] then ynh_abort_if_errors fi diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7488a1646..ba4351c2f 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -627,7 +627,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if upgrade_failed or broke_the_system: # display this if there are remaining apps - if apps[number + 1 :]: + if apps[number + 1:]: not_upgraded_apps = apps[number:] logger.error( m18n.n( @@ -2048,9 +2048,9 @@ def _set_default_ask_questions(questions, script_name="install"): # Also it in fact doesn't make sense for any of those questions to have an example value nor a default value... if question.get("type") in ["domain", "user", "password"]: - if "example" in arg: + if "example" in question: del question["example"] - if "default" in arg: + if "default" in question: del question["domain"] return questions From 8a71fae732212a28b215b1a20f12b49da9dc7958 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 1 Nov 2021 22:41:31 +0100 Subject: [PATCH 08/54] manifestv2 upgrade: implement safety backup mecanism in the core --- src/yunohost/app.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index ba4351c2f..05db1fa56 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -469,6 +469,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False from yunohost.permission import permission_sync_to_user from yunohost.regenconf import manually_modified_files from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers + from yunohost.backup import backup_list, backup_create, backup_delete, backup_restore apps = app # Check if disk space available @@ -556,6 +557,31 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False # Check requirements _check_manifest_requirements(manifest) + + if manifest["packaging_format"] >= 2: + if no_safety_backup: + logger.warning("Skipping the creation of a backup prior to the upgrade.") + else: + # FIXME: i18n + logger.info("Creating a safety backup prior to the upgrade") + + # Switch between pre-upgrade1 or pre-upgrade2 + safety_backup_name = f"{app_instance_name}-pre-upgrade1" + other_safety_backup_name = f"{app_instance_name}-pre-upgrade2" + if safety_backup_name in backup_list()["archives"]: + safety_backup_name = f"{app_instance_name}-pre-upgrade2" + other_safety_backup_name = f"{app_instance_name}-pre-upgrade1" + + backup_create(name=safety_backup_name, apps=[app_instance_name]) + + if safety_backup_name in backup_list()["archives"]: + # if the backup suceeded, delete old safety backup to save space + if other_safety_backup_name in backup_list()["archives"]: + backup_delete(other_safety_backup_name) + else: + # Is this needed ? Shouldn't backup_create report an expcetion if backup failed ? + raise YunohostError("Uhoh the safety backup failed ?! Aborting the upgrade process.", raw_msg=True) + _assert_system_is_sane_for_app(manifest, "pre") app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) @@ -567,7 +593,8 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False env_dict["YNH_APP_UPGRADE_TYPE"] = upgrade_type env_dict["YNH_APP_MANIFEST_VERSION"] = str(app_new_version) env_dict["YNH_APP_CURRENT_VERSION"] = str(app_current_version) - env_dict["NO_BACKUP_UPGRADE"] = "1" if no_safety_backup else "0" + if manifest["packaging_format"] < 2: + env_dict["NO_BACKUP_UPGRADE"] = "1" if no_safety_backup else "0" # We'll check that the app didn't brutally edit some system configuration manually_modified_files_before_install = manually_modified_files() @@ -599,6 +626,16 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False ), ) finally: + + # If upgrade failed, try to restore the safety backup + if upgrade_failed and manifest["packaging_format"] >= 2 and not no_safety_backup: + logger.warning("Upgrade failed ... attempting to restore the satefy backup (Yunohost first need to remove the app for this) ...") + + app_remove(app_instance_name) + backup_restore(name=safety_backup_name, apps=[app_instance_name], force=True) + if not _is_installed(app_instance_name): + logger.error("Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|") + # Whatever happened (install success or failure) we check if it broke the system # and warn the user about it try: From 0a750b7b61f2d0c01a90d3049a928c6963ab0dff Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Nov 2021 16:36:38 +0100 Subject: [PATCH 09/54] Absolutely yolo iteration on app resources draft --- src/yunohost/app.py | 48 ++++++----- src/yunohost/utils/config.py | 20 ++++- src/yunohost/utils/resources.py | 144 +++++++++++++++++++++++++------- 3 files changed, 160 insertions(+), 52 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 05db1fa56..8e05c37c9 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -59,6 +59,7 @@ from yunohost.utils.config import ( DomainQuestion, PathQuestion, ) +from yunohost.utils.resources import AppResourceSet from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory @@ -803,6 +804,7 @@ def app_install( confirm_install(app) manifest, extracted_app_folder = _extract_app(app) + packaging_format = manifest["packaging_format"] # Check ID if "id" not in manifest or "__" in manifest["id"] or "." in manifest["id"]: @@ -827,7 +829,7 @@ def app_install( app_instance_name = app_id # Retrieve arguments list for install script - raw_questions = manifest.get("arguments", {}).get("install", {}) + raw_questions = manifest["install"] questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args) args = { question.name: question.value @@ -836,11 +838,12 @@ def app_install( } # Validate domain / path availability for webapps - path_requirement = _guess_webapp_path_requirement(extracted_app_folder) - _validate_webpath_requirement(args, path_requirement) + if packaging_format < 2: + path_requirement = _guess_webapp_path_requirement(extracted_app_folder) + _validate_webpath_requirement(args, path_requirement) - # Attempt to patch legacy helpers ... - _patch_legacy_helpers(extracted_app_folder) + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(extracted_app_folder) # Apply dirty patch to make php5 apps compatible with php7 _patch_legacy_php_versions(extracted_app_folder) @@ -870,7 +873,6 @@ def app_install( } # If packaging_format v2+, save all install questions as settings - packaging_format = int(manifest.get("packaging_format", 0)) if packaging_format >= 2: for arg_name, arg_value in args.items(): @@ -891,17 +893,23 @@ def app_install( recursive=True, ) - # Initialize the main permission for the app - # The permission is initialized with no url associated, and with tile disabled - # For web app, the root path of the app will be added as url and the tile - # will be enabled during the app install. C.f. 'app_register_url()' below. - permission_create( - app_instance_name + ".main", - allowed=["all_users"], - label=label, - show_tile=False, - protected=False, - ) + + resources = AppResourceSet(manifest["resources"], app_instance_name) + resources.check_availability() + resources.provision() + + if packaging_format < 2: + # Initialize the main permission for the app + # The permission is initialized with no url associated, and with tile disabled + # For web app, the root path of the app will be added as url and the tile + # will be enabled during the app install. C.f. 'app_register_url()' below. + permission_create( + app_instance_name + ".main", + allowed=["all_users"], + label=label, + show_tile=False, + protected=False, + ) # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( @@ -2354,13 +2362,13 @@ def _guess_webapp_path_requirement(app_folder: str) -> str: # is an available url and normalize the path. manifest = _get_manifest_of_app(app_folder) - raw_questions = manifest.get("arguments", {}).get("install", {}) + raw_questions = manifest["install"] domain_questions = [ - question for question in raw_questions if question.get("type") == "domain" + question for question in raw_questions.values() if question.get("type") == "domain" ] path_questions = [ - question for question in raw_questions if question.get("type") == "path" + question for question in raw_questions.values() if question.get("type") == "path" ] if len(domain_questions) == 0 and len(path_questions) == 0: diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 4ee62c6f7..b7bd4e723 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -1052,6 +1052,21 @@ class UserQuestion(Question): break +class GroupQuestion(Question): + argument_type = "group" + + def __init__(self, question, context: Mapping[str, Any] = {}): + + from yunohost.user import user_group_list + + super().__init__(question, context) + + self.choices = list(user_group_list(short=True)["groups"]) + + if self.default is None: + self.default = "all_users" + + class NumberQuestion(Question): argument_type = "number" default_value = None @@ -1204,6 +1219,7 @@ ARGUMENTS_TYPE_PARSERS = { "boolean": BooleanQuestion, "domain": DomainQuestion, "user": UserQuestion, + "group": GroupQuestion, "number": NumberQuestion, "range": NumberQuestion, "display_text": DisplayTextQuestion, @@ -1243,9 +1259,9 @@ def ask_questions_and_parse_answers( out = [] - for raw_question in raw_questions: + for name, raw_question in raw_questions.items(): question_class = ARGUMENTS_TYPE_PARSERS[raw_question.get("type", "string")] - raw_question["value"] = answers.get(raw_question["name"]) + raw_question["value"] = answers.get(name) question = question_class(raw_question, context=answers) answers[question.name] = question.ask_if_needed() out.append(question) diff --git a/src/yunohost/utils/resources.py b/src/yunohost/utils/resources.py index 42ae94d83..c71af09b8 100644 --- a/src/yunohost/utils/resources.py +++ b/src/yunohost/utils/resources.py @@ -27,15 +27,37 @@ from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory -class AppResource: +class AppResource(object): - def __init__(self, properties: Dict[str, Any], app_id: str, app_settings): + def __init__(self, properties: Dict[str, Any], app_id: str): + + self.app_id = app_id for key, value in self.default_properties.items(): setattr(self, key, value) - for key, value in properties: - setattr(self. key, value) + for key, value in properties.items(): + setattr(self, key, value) + + def get_app_settings(self): + from yunohost.app import _get_app_settings + return _get_app_settings(self.app_id) + + def check_availability(self, context: Dict): + pass + + +class AppResourceSet: + + def __init__(self, resources_dict: Dict[str, Dict[str, Any]], app_id: str): + + self.set = {name: AppResourceClassesByType[name](infos, app_id) + for name, infos in resources_dict.items()} + + def check_availability(self): + + for name, resource in self.set.items(): + resource.check_availability(context={}) M = 1024 ** 2 @@ -68,19 +90,16 @@ class DiskAppResource(AppResource): } def __init__(self, *args, **kwargs): - super().__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) # FIXME: better error handling assert self.space in sizes - def provision_or_update(self, context: Dict): + def assert_availability(self, context: Dict): if free_space_in_directory("/") <= sizes[self.space] \ or free_space_in_directory("/var") <= sizes[self.space]: raise YunohostValidationError("Not enough disk space") # FIXME: i18n / better messaging - def deprovision(self, context: Dict): - pass - class RamAppResource(AppResource): type = "ram" @@ -92,13 +111,13 @@ class RamAppResource(AppResource): } def __init__(self, *args, **kwargs): - super().__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) # FIXME: better error handling assert self.build in sizes assert self.runtime in sizes assert isinstance(self.include_swap, bool) - def provision_or_update(self, context: Dict): + def assert_availability(self, context: Dict): memory = psutil.virtual_memory().available if self.include_swap: @@ -109,37 +128,78 @@ class RamAppResource(AppResource): if memory <= max_size: raise YunohostValidationError("Not enough RAM/swap") # FIXME: i18n / better messaging - def deprovision(self, context: Dict): + +class AptDependenciesAppResource(AppResource): + type = "apt" + + default_properties = { + "packages": [], + "extras": {} + } + + def check_availability(self, context): + # ? FIXME + # call helpers idk ... + pass + +class SourcesAppResource(AppResource): + type = "sources" + + default_properties = { + "main": {"url": "?", "sha256sum": "?", "predownload": True} + } + + def check_availability(self, context): + # ? FIXME + # call request.head on the url idk pass -class WebpathAppResource(AppResource): - type = "webpath" +class RoutesAppResource(AppResource): + type = "routes" default_properties = { - "url": "__DOMAIN____PATH__" + "full_domain": False, + "main": { + "url": "/", + "additional_urls": [], + "init_allowed": "__FIXME__", + "show_tile": True, + "protected": False, + "auth_header": True, + "label": "FIXME", + } } - def provision_or_update(self, context: Dict): + def check_availability(self, context): from yunohost.app import _assert_no_conflicting_apps - # Check the url is available - domain = context["app_settings"]["domain"] - path = context["app_settings"]["path"] or "/" - _assert_no_conflicting_apps(domain, path, ignore_app=context["app"]) - context["app_settings"]["path"] = path + app_settings = self.get_app_settings() + domain = app_settings["domain"] + path = app_settings["path"] if not self.full_domain else "/" + _assert_no_conflicting_apps(domain, path, ignore_app=self.app_id) + + def provision_or_update(self, context: Dict): if context["app_action"] == "install": + pass # FIXME # Initially, the .main permission is created with no url at all associated # When the app register/books its web url, we also add the url '/' # (meaning the root of the app, domain.tld/path/) # and enable the tile to the SSO, and both of this should match 95% of apps # For more specific cases, the app is free to change / add urls or disable # the tile using the permission helpers. - permission_url(app + ".main", url="/", sync_perm=False) - user_permission_update(app + ".main", show_tile=True, sync_perm=False) - permission_sync_to_user() + #permission_create( + # self.app_id + ".main", + # allowed=["all_users"], + # label=label, + # show_tile=False, + # protected=False, + #) + #permission_url(app + ".main", url="/", sync_perm=False) + #user_permission_update(app + ".main", show_tile=True, sync_perm=False) + #permission_sync_to_user() def deprovision(self, context: Dict): del context["app_settings"]["domain"] @@ -177,8 +237,8 @@ class PortAppResource(AppResource): raise NotImplementedError() -class UserAppResource(AppResource): - type = "user" +class SystemuserAppResource(AppResource): + type = "system_user" default_properties = { "username": "__APP__", @@ -187,6 +247,12 @@ class UserAppResource(AppResource): "groups": [] } + def check_availability(self, context): + if os.system(f"getent passwd {self.username} &>/dev/null") != 0: + raise YunohostValidationError(f"User {self.username} already exists") + if os.system(f"getent group {self.username} &>/dev/null") != 0: + raise YunohostValidationError(f"Group {self.username} already exists") + def provision_or_update(self, context: str): raise NotImplementedError() @@ -195,13 +261,19 @@ class UserAppResource(AppResource): class InstalldirAppResource(AppResource): - type = "installdir" + type = "install_dir" default_properties = { - "dir": "/var/www/__APP__", + "dir": "/var/www/__APP__", # FIXME or choose to move this elsewhere nowadays idk... "alias": "final_path" } + # FIXME: change default dir to /opt/stuff if app ain't a webapp ... + + def check_availability(self, context): + if os.path.exists(self.dir): + raise YunohostValidationError(f"Folder {self.dir} already exists") + def provision_or_update(self, context: Dict): if context["app_action"] in ["install", "restore"]: @@ -218,12 +290,16 @@ class InstalldirAppResource(AppResource): class DatadirAppResource(AppResource): - type = "datadir" + type = "data_dir" default_properties = { - "dir": "/home/yunohost.app/__APP__", + "dir": "/home/yunohost.app/__APP__", # FIXME or choose to move this elsewhere nowadays idk... } + def check_availability(self, context): + if os.path.exists(self.dir): + raise YunohostValidationError(f"Folder {self.dir} already exists") + def provision_or_update(self, context: Dict): if "datadir" not in context["app_settings"]: context["app_settings"]["datadir"] = self.dir @@ -240,8 +316,16 @@ class DBAppResource(AppResource): "type": "mysql" } + def check_availability(self, context): + # FIXME : checking availability sort of imply that mysql / postgresql is installed + # or we gotta make sure mariadb-server or postgresql is gonna be installed (apt resource) + pass + def provision_or_update(self, context: str): raise NotImplementedError() def deprovision(self, context: Dict): raise NotImplementedError() + + +AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()} From 6a437c0b4fe80149328bfa2150af4525f4a3c098 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Nov 2021 18:33:00 +0100 Subject: [PATCH 10/54] Rework requirement checks to integrate architecture, multiinnstance, disk/ram, ... + drop disk/ram as resource, have them directly in 'integration' --- src/yunohost/app.py | 125 ++++++++++++++++++++------------ src/yunohost/backup.py | 1 + src/yunohost/utils/resources.py | 71 ------------------ 3 files changed, 79 insertions(+), 118 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index c5d75b25b..d555b6872 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -35,6 +35,7 @@ import tempfile import copy from collections import OrderedDict from typing import List, Tuple, Dict, Any +from packaging import version from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger @@ -52,7 +53,7 @@ from moulinette.utils.filesystem import ( chmod, ) -from yunohost.utils import packages +from yunohost.utils.packages import dpkg_is_broken, get_ynh_package_version from yunohost.utils.config import ( ConfigPanel, ask_questions_and_parse_answers, @@ -194,7 +195,6 @@ def app_info(app, full=False): def _app_upgradable(app_infos): - from packaging import version # Determine upgradability @@ -460,7 +460,6 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False no_safety_backup -- Disable the safety backup during upgrade """ - from packaging import version from yunohost.hook import ( hook_add, hook_remove, @@ -557,7 +556,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False upgrade_type = "UPGRADE_FULL" # Check requirements - _check_manifest_requirements(manifest) + _check_manifest_requirements(manifest, action="upgrade") if manifest["packaging_format"] >= 2: if no_safety_backup: @@ -814,15 +813,12 @@ def app_install( label = label if label else manifest["name"] # Check requirements - _check_manifest_requirements(manifest) + _check_manifest_requirements(manifest, action="install") _assert_system_is_sane_for_app(manifest, "pre") # Check if app can be forked instance_number = _next_instance_number_for_app(app_id) if instance_number > 1: - if "multi_instance" not in manifest or not is_true(manifest["multi_instance"]): - raise YunohostValidationError("app_already_installed", app=app_id) - # Change app_id to the forked app id app_instance_name = app_id + "__" + str(instance_number) else: @@ -1995,11 +1991,13 @@ def _convert_v1_manifest_to_v2(manifest): manifest["upstream"]["website"] = manifest["url"] manifest["integration"] = { - "yunohost": manifest.get("requirements", {}).get("yunohost"), + "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""), "architectures": "all", - "multi_instance": manifest.get("multi_instance", False), + "multi_instance": is_true(manifest.get("multi_instance", False)), "ldap": "?", "sso": "?", + "disk": "50M", + "ram": {"build": "50M", "runtime": "10M", "include_swap": False} } maintainer = manifest.get("maintainer", {}).get("name") @@ -2017,29 +2015,12 @@ def _convert_v1_manifest_to_v2(manifest): manifest["install"][name] = question manifest["resources"] = { - "disk": { - "build": "50M", # This is an *estimate* minimum value for the disk needed at build time (e.g. during install/upgrade) and during regular usage - "usage": "50M" # Please only use round values such as: 10M, 100M, 200M, 400M, 800M, 1G, 2G, 4G, 8G - }, - "ram": { - "build": "50M", # This is an *estimate* minimum value for the RAM needed at build time (i.e. during install/upgrade) and during regular usage - "usage": "10M", # Please only use round values like ["10M", "100M", "200M", "400M", "800M", "1G", "2G", "4G", "8G"] - "include_swap": False - }, - "route": {}, + "system_user": {}, "install_dir": { - "base_dir": "/var/www/", # This means that the app shall be installed in /var/www/$app which is the standard for webapps. You may change this to /opt/ if the app is a system app. "alias": "final_path" } } - if "domain" in manifest["install"] and "path" in manifest["install"]: - manifest["resources"]["route"]["url"] = "{domain}{path}" - elif "path" not in manifest["install"]: - manifest["resources"]["route"]["url"] = "{domain}/" - else: - del manifest["resources"]["route"] - keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] @@ -2325,32 +2306,60 @@ def _get_all_installed_apps_id(): return all_apps_ids_formatted -def _check_manifest_requirements(manifest: Dict): +def _check_manifest_requirements(manifest: Dict, action: str): """Check if required packages are met from the manifest""" if manifest["packaging_format"] not in [1, 2]: raise YunohostValidationError("app_packaging_format_not_supported") - requirements = manifest.get("requirements", dict()) - - if not requirements: - return - - app = manifest.get("id", "?") + app_id = manifest["id"] logger.debug(m18n.n("app_requirements_checking", app=app)) - # Iterate over requirements - for pkgname, spec in requirements.items(): - if not packages.meets_version_specifier(pkgname, spec): - version = packages.ynh_packages_version()[pkgname]["version"] - raise YunohostValidationError( - "app_requirements_unmeet", - pkgname=pkgname, - version=version, - spec=spec, - app=app, - ) + # Yunohost version requirement + + yunohost_requirement = version.parse(manifest["integration"]["yunohost"] or "4.3") + yunohost_installed_version = get_ynh_package_version("yunohost")["version"] + if yunohost_requirement > yunohost_installed_version: + # FIXME : i18n + raise YunohostValidationError(f"This app requires Yunohost >= {yunohost_requirement} but current installed version is {yunohost_installed_version}") + + # Architectures + arch_requirement = manifest["integration"]["architectures"] + if arch_requirement != "all": + arch = check_output("dpkg --print-architecture") + if arch not in arch_requirement: + # FIXME: i18n + raise YunohostValidationError(f"This app can only be installed on architectures {', '.join(arch_requirement)} but your server architecture is {arch}") + + # Multi-instance + if action == "install" and manifest["integration"]["multi_instance"] == False: + apps = _installed_apps() + sibling_apps = [a for a in apps if a == app_id or a.startswith(f"{app_id}__")] + if len(sibling_apps) > 0: + raise YunohostValidationError("app_already_installed", app=app_id) + + # Disk + if action == "install": + disk_requirement = manifest["integration"]["disk"] + + if free_space_in_directory("/") <= human_to_binary[disk_requirement] \ + or free_space_in_directory("/var") <= human_to_binary[disk_requirement]: + # FIXME : i18m + raise YunohostValidationError("This app requires {disk_requirement} free space.") + + # Ram for build + import psutil + ram_build_requirement = manifest["integration"]["ram"]["build"] + ram_include_swap = manifest["integration"]["ram"]["include_swap"] + + ram_available = psutil.virtual_memory().available + if ram_include_swap: + ram_available += psutil.swap_memory().available + + if ram_available < human_to_binary(ram_build_requirement): + # FIXME : i18n + raise YunohostValidationError("This app requires {ram_build_requirement} RAM available to install/upgrade") def _guess_webapp_path_requirement(app_folder: str) -> str: @@ -2688,8 +2697,30 @@ def _assert_system_is_sane_for_app(manifest, when): "app_action_broke_system", services=", ".join(faulty_services) ) - if packages.dpkg_is_broken(): + if dpkg_is_broken(): if when == "pre": raise YunohostValidationError("dpkg_is_broken") elif when == "post": raise YunohostError("this_action_broke_dpkg") + + + +def human_to_binary(size: str) -> int: + + symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y") + factor = {} + for i, s in enumerate(symbols): + factor[s] = 1 << (i + 1) * 10 + + suffix = size[-1] + size = size[:-1] + + if suffix not in symbols: + raise YunohostError(f"Invalid size suffix '{suffix}', expected one of {symbols}") + + try: + size = float(size) + except Exception: + raise YunohostError(f"Failed to convert size {size} to float") + + return size * factor[suffix] diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 8deb2b3bf..78aab0dce 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2695,3 +2695,4 @@ def binary_to_human(n, customary=False): value = float(n) / prefix[s] return "%.1f%s" % (value, s) return "%s" % n + diff --git a/src/yunohost/utils/resources.py b/src/yunohost/utils/resources.py index c71af09b8..424e38f91 100644 --- a/src/yunohost/utils/resources.py +++ b/src/yunohost/utils/resources.py @@ -20,11 +20,9 @@ """ import os import copy -import psutil from typing import Dict, Any from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.filesystem import free_space_in_directory class AppResource(object): @@ -60,75 +58,6 @@ class AppResourceSet: resource.check_availability(context={}) -M = 1024 ** 2 -G = 1024 * M -sizes = { - "10M": 10 * M, - "20M": 20 * M, - "40M": 40 * M, - "80M": 80 * M, - "100M": 100 * M, - "200M": 200 * M, - "400M": 400 * M, - "800M": 800 * M, - "1G": 1 * G, - "2G": 2 * G, - "4G": 4 * G, - "8G": 8 * G, - "10G": 10 * G, - "20G": 20 * G, - "40G": 40 * G, - "80G": 80 * G, -} - - -class DiskAppResource(AppResource): - type = "disk" - - default_properties = { - "space": "10M", - } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # FIXME: better error handling - assert self.space in sizes - - def assert_availability(self, context: Dict): - - if free_space_in_directory("/") <= sizes[self.space] \ - or free_space_in_directory("/var") <= sizes[self.space]: - raise YunohostValidationError("Not enough disk space") # FIXME: i18n / better messaging - - -class RamAppResource(AppResource): - type = "ram" - - default_properties = { - "build": "10M", - "runtime": "10M", - "include_swap": False - } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # FIXME: better error handling - assert self.build in sizes - assert self.runtime in sizes - assert isinstance(self.include_swap, bool) - - def assert_availability(self, context: Dict): - - memory = psutil.virtual_memory().available - if self.include_swap: - memory += psutil.swap_memory().available - - max_size = max(sizes[self.build], sizes[self.runtime]) - - if memory <= max_size: - raise YunohostValidationError("Not enough RAM/swap") # FIXME: i18n / better messaging - - class AptDependenciesAppResource(AppResource): type = "apt" From 810534c661f5c57050c974a8bd8ddd2c7c46bc74 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Nov 2021 20:38:51 +0100 Subject: [PATCH 11/54] Cleanup / refactor a bunch of utils into utils/system.py --- data/actionsmap/yunohost.yml | 2 +- data/hooks/diagnosis/00-basesystem.py | 13 +- src/yunohost/app.py | 82 ++++--------- src/yunohost/backup.py | 88 +++++--------- .../data_migrations/0015_migrate_to_buster.py | 4 +- .../0017_postgresql_9p6_to_11.py | 2 +- src/yunohost/log.py | 2 +- src/yunohost/tools.py | 23 +--- src/yunohost/user.py | 12 +- src/yunohost/utils/filesystem.py | 31 ----- src/yunohost/utils/{packages.py => system.py} | 111 ++++++++++++------ 11 files changed, 146 insertions(+), 224 deletions(-) delete mode 100644 src/yunohost/utils/filesystem.py rename src/yunohost/utils/{packages.py => system.py} (67%) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 16ea2c5d2..e63a04a41 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -43,7 +43,7 @@ _global: help: Display YunoHost packages versions action: callback callback: - method: yunohost.utils.packages.ynh_packages_version + method: yunohost.utils.system.ynh_packages_version return: true ############################# diff --git a/data/hooks/diagnosis/00-basesystem.py b/data/hooks/diagnosis/00-basesystem.py index b472a2d32..afb44ff37 100644 --- a/data/hooks/diagnosis/00-basesystem.py +++ b/data/hooks/diagnosis/00-basesystem.py @@ -7,7 +7,11 @@ import subprocess from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file, read_json, write_to_json from yunohost.diagnosis import Diagnoser -from yunohost.utils.packages import ynh_packages_version +from yunohost.utils.system import ( + ynh_packages_version, + system_virt, + system_arch, +) class BaseSystemDiagnoser(Diagnoser): @@ -18,15 +22,12 @@ class BaseSystemDiagnoser(Diagnoser): def run(self): - # Detect virt technology (if not bare metal) and arch - # Gotta have this "|| true" because it systemd-detect-virt return 'none' - # with an error code on bare metal ~.~ - virt = check_output("systemd-detect-virt || true", shell=True) + virt = system_virt() if virt.lower() == "none": virt = "bare-metal" # Detect arch - arch = check_output("dpkg --print-architecture") + arch = system_arch() hardware = dict( meta={"test": "hardware"}, status="INFO", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d555b6872..e251b99de 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -53,7 +53,6 @@ from moulinette.utils.filesystem import ( chmod, ) -from yunohost.utils.packages import dpkg_is_broken, get_ynh_package_version from yunohost.utils.config import ( ConfigPanel, ask_questions_and_parse_answers, @@ -63,7 +62,15 @@ from yunohost.utils.config import ( from yunohost.utils.resources import AppResourceSet from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.filesystem import free_space_in_directory +from yunohost.utils.system import ( + free_space_in_directory, + dpkg_is_broken, + get_ynh_package_version, + system_arch, + human_to_binary, + binary_to_human, + ram_available, +) from yunohost.log import is_unit_operation, OperationLogger from yunohost.app_catalog import ( # noqa app_catalog, @@ -179,9 +186,7 @@ def app_info(app, full=False): ret["supports_backup_restore"] = os.path.exists( os.path.join(setting_path, "scripts", "backup") ) and os.path.exists(os.path.join(setting_path, "scripts", "restore")) - ret["supports_multi_instance"] = is_true( - local_manifest.get("integration", {}).get("multi_instance", False) - ) + ret["supports_multi_instance"] = local_manifest.get("integration", {}).get("multi_instance", False) ret["supports_config_panel"] = os.path.exists( os.path.join(setting_path, "config_panel.toml") ) @@ -1993,7 +1998,7 @@ def _convert_v1_manifest_to_v2(manifest): manifest["integration"] = { "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""), "architectures": "all", - "multi_instance": is_true(manifest.get("multi_instance", False)), + "multi_instance": manifest.get("multi_instance", False), "ldap": "?", "sso": "?", "disk": "50M", @@ -2314,12 +2319,12 @@ def _check_manifest_requirements(manifest: Dict, action: str): app_id = manifest["id"] - logger.debug(m18n.n("app_requirements_checking", app=app)) + logger.debug(m18n.n("app_requirements_checking", app=app_id)) # Yunohost version requirement yunohost_requirement = version.parse(manifest["integration"]["yunohost"] or "4.3") - yunohost_installed_version = get_ynh_package_version("yunohost")["version"] + yunohost_installed_version = version.parse(get_ynh_package_version("yunohost")["version"]) if yunohost_requirement > yunohost_installed_version: # FIXME : i18n raise YunohostValidationError(f"This app requires Yunohost >= {yunohost_requirement} but current installed version is {yunohost_installed_version}") @@ -2327,7 +2332,7 @@ def _check_manifest_requirements(manifest: Dict, action: str): # Architectures arch_requirement = manifest["integration"]["architectures"] if arch_requirement != "all": - arch = check_output("dpkg --print-architecture") + arch = system_arch() if arch not in arch_requirement: # FIXME: i18n raise YunohostValidationError(f"This app can only be installed on architectures {', '.join(arch_requirement)} but your server architecture is {arch}") @@ -2343,23 +2348,23 @@ def _check_manifest_requirements(manifest: Dict, action: str): if action == "install": disk_requirement = manifest["integration"]["disk"] - if free_space_in_directory("/") <= human_to_binary[disk_requirement] \ - or free_space_in_directory("/var") <= human_to_binary[disk_requirement]: + if free_space_in_directory("/") <= human_to_binary(disk_requirement) \ + or free_space_in_directory("/var") <= human_to_binary(disk_requirement): # FIXME : i18m - raise YunohostValidationError("This app requires {disk_requirement} free space.") + raise YunohostValidationError(f"This app requires {disk_requirement} free space.") # Ram for build - import psutil ram_build_requirement = manifest["integration"]["ram"]["build"] ram_include_swap = manifest["integration"]["ram"]["include_swap"] - ram_available = psutil.virtual_memory().available + ram, swap = ram_available() if ram_include_swap: - ram_available += psutil.swap_memory().available + ram += swap - if ram_available < human_to_binary(ram_build_requirement): + if ram < human_to_binary(ram_build_requirement): # FIXME : i18n - raise YunohostValidationError("This app requires {ram_build_requirement} RAM available to install/upgrade") + ram_human = binary_to_human(ram) + raise YunohostValidationError(f"This app requires {ram_build_requirement} RAM to install/upgrade but only {ram_human} is available right now.") def _guess_webapp_path_requirement(app_folder: str) -> str: @@ -2499,7 +2504,7 @@ def _make_environment_for_app_script( "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), "YNH_APP_MANIFEST_VERSION": manifest.get("version", "?"), "YNH_APP_PACKAGING_FORMAT": str(manifest["packaging_format"]), - "YNH_ARCH": check_output("dpkg --print-architecture"), + "YNH_ARCH": system_arch(), } if workdir: @@ -2607,26 +2612,6 @@ def _make_tmp_workdir_for_app(app=None): return tmpdir -def is_true(arg): - """ - Convert a string into a boolean - - Keyword arguments: - arg -- The string to convert - - Returns: - Boolean - - """ - if isinstance(arg, bool): - return arg - elif isinstance(arg, str): - return arg.lower() in ["yes", "true", "on"] - else: - logger.debug("arg should be a boolean or a string, got %r", arg) - return True if arg else False - - def unstable_apps(): output = [] @@ -2703,24 +2688,3 @@ def _assert_system_is_sane_for_app(manifest, when): elif when == "post": raise YunohostError("this_action_broke_dpkg") - - -def human_to_binary(size: str) -> int: - - symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y") - factor = {} - for i, s in enumerate(symbols): - factor[s] = 1 << (i + 1) * 10 - - suffix = size[-1] - size = size[:-1] - - if suffix not in symbols: - raise YunohostError(f"Invalid size suffix '{suffix}', expected one of {symbols}") - - try: - size = float(size) - except Exception: - raise YunohostError(f"Failed to convert size {size} to float") - - return size * factor[suffix] diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 78aab0dce..67b5cde05 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -39,9 +39,8 @@ from functools import reduce from packaging import version from moulinette import Moulinette, m18n -from moulinette.utils import filesystem from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml +from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml, rm, chown, chmod from moulinette.utils.process import check_output import yunohost.domain @@ -67,8 +66,12 @@ from yunohost.tools import ( from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger, is_unit_operation from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.packages import ynh_packages_version -from yunohost.utils.filesystem import free_space_in_directory +from yunohost.utils.system import ( + free_space_in_directory, + get_ynh_package_version, + binary_to_human, + space_used_by_directory, +) from yunohost.settings import settings_get BACKUP_PATH = "/home/yunohost.backup" @@ -312,7 +315,7 @@ class BackupManager: "size_details": self.size_details, "apps": self.apps_return, "system": self.system_return, - "from_yunohost_version": ynh_packages_version()["yunohost"]["version"], + "from_yunohost_version": get_ynh_package_version("yunohost")["version"], } @property @@ -342,7 +345,7 @@ class BackupManager: # FIXME replace isdir by exists ? manage better the case where the path # exists if not os.path.isdir(self.work_dir): - filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin") + mkdir(self.work_dir, 0o750, parents=True, uid="admin") elif self.is_tmp_work_dir: logger.debug( @@ -357,8 +360,8 @@ class BackupManager: # If umount succeeded, remove the directory (we checked that # we're in /home/yunohost.backup/tmp so that should be okay... # c.f. method clean() which also does this) - filesystem.rm(self.work_dir, recursive=True, force=True) - filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin") + rm(self.work_dir, recursive=True, force=True) + mkdir(self.work_dir, 0o750, parents=True, uid="admin") # # Backup target management # @@ -535,7 +538,7 @@ class BackupManager: successfull_system = self.targets.list("system", include=["Success", "Warning"]) if not successfull_apps and not successfull_system: - filesystem.rm(self.work_dir, True, True) + rm(self.work_dir, True, True) raise YunohostError("backup_nothings_done") # Add unlisted files from backup tmp dir @@ -647,7 +650,7 @@ class BackupManager: restore_hooks_dir = os.path.join(self.work_dir, "hooks", "restore") if not os.path.exists(restore_hooks_dir): - filesystem.mkdir(restore_hooks_dir, mode=0o700, parents=True, uid="root") + mkdir(restore_hooks_dir, mode=0o700, parents=True, uid="root") restore_hooks = hook_list("restore")["hooks"] @@ -714,7 +717,7 @@ class BackupManager: tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app) try: # Prepare backup directory for the app - filesystem.mkdir(tmp_app_bkp_dir, 0o700, True, uid="root") + mkdir(tmp_app_bkp_dir, 0o700, True, uid="root") # Copy the app settings to be able to call _common.sh shutil.copytree(app_setting_path, settings_dir) @@ -753,7 +756,7 @@ class BackupManager: # Remove tmp files in all situations finally: shutil.rmtree(tmp_workdir_for_app) - filesystem.rm(env_dict["YNH_BACKUP_CSV"], force=True) + rm(env_dict["YNH_BACKUP_CSV"], force=True) # # Actual backup archive creation / method management # @@ -796,7 +799,7 @@ class BackupManager: if row["dest"] == "info.json": continue - size = disk_usage(row["source"]) + size = space_used_by_directory(row["source"], follow_symlinks=False) # Add size to apps details splitted_dest = row["dest"].split("/") @@ -945,7 +948,7 @@ class RestoreManager: ret = subprocess.call(["umount", self.work_dir]) if ret != 0: logger.warning(m18n.n("restore_cleaning_failed")) - filesystem.rm(self.work_dir, recursive=True, force=True) + rm(self.work_dir, recursive=True, force=True) # # Restore target manangement # @@ -975,7 +978,7 @@ class RestoreManager: available_restore_system_hooks = hook_list("restore")["hooks"] custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, "restore") - filesystem.mkdir(custom_restore_hook_folder, 755, parents=True, force=True) + mkdir(custom_restore_hook_folder, 755, parents=True, force=True) for system_part in target_list: # By default, we'll use the restore hooks on the current install @@ -1080,7 +1083,7 @@ class RestoreManager: else: raise YunohostError("restore_removing_tmp_dir_failed") - filesystem.mkdir(self.work_dir, parents=True) + mkdir(self.work_dir, parents=True) self.method.mount() @@ -1398,7 +1401,7 @@ class RestoreManager: # Delete _common.sh file in backup common_file = os.path.join(app_backup_in_archive, "_common.sh") - filesystem.rm(common_file, force=True) + rm(common_file, force=True) # Check if the app has a restore script app_restore_script_in_archive = os.path.join(app_scripts_in_archive, "restore") @@ -1414,14 +1417,14 @@ class RestoreManager: ) app_scripts_new_path = os.path.join(app_settings_new_path, "scripts") shutil.copytree(app_settings_in_archive, app_settings_new_path) - filesystem.chmod(app_settings_new_path, 0o400, 0o400, True) - filesystem.chown(app_scripts_new_path, "root", None, True) + chmod(app_settings_new_path, 0o400, 0o400, True) + chown(app_scripts_new_path, "root", None, True) # Copy the app scripts to a writable temporary folder tmp_workdir_for_app = _make_tmp_workdir_for_app() copytree(app_scripts_in_archive, tmp_workdir_for_app) - filesystem.chmod(tmp_workdir_for_app, 0o700, 0o700, True) - filesystem.chown(tmp_workdir_for_app, "root", None, True) + chmod(tmp_workdir_for_app, 0o700, 0o700, True) + chown(tmp_workdir_for_app, "root", None, True) restore_script = os.path.join(tmp_workdir_for_app, "restore") # Restore permissions @@ -1724,7 +1727,7 @@ class BackupMethod(object): raise YunohostError("backup_cleaning_failed") if self.manager.is_tmp_work_dir: - filesystem.rm(self.work_dir, True, True) + rm(self.work_dir, True, True) def _check_is_enough_free_space(self): """ @@ -1772,11 +1775,11 @@ class BackupMethod(object): # Be sure the parent dir of destination exists if not os.path.isdir(dest_dir): - filesystem.mkdir(dest_dir, parents=True) + mkdir(dest_dir, parents=True) # For directory, attempt to mount bind if os.path.isdir(src): - filesystem.mkdir(dest, parents=True, force=True) + mkdir(dest, parents=True, force=True) try: subprocess.check_call(["mount", "--rbind", src, dest]) @@ -1830,7 +1833,7 @@ class BackupMethod(object): # to mounting error # Compute size to copy - size = sum(disk_usage(path["source"]) for path in paths_needed_to_be_copied) + size = sum(space_used_by_directory(path["source"], follow_symlinks=False) for path in paths_needed_to_be_copied) size /= 1024 * 1024 # Convert bytes to megabytes # Ask confirmation for copying @@ -1882,7 +1885,7 @@ class CopyBackupMethod(BackupMethod): dest_parent = os.path.dirname(dest) if not os.path.exists(dest_parent): - filesystem.mkdir(dest_parent, 0o700, True, uid="admin") + mkdir(dest_parent, 0o700, True, uid="admin") if os.path.isdir(source): shutil.copytree(source, dest) @@ -1900,7 +1903,7 @@ class CopyBackupMethod(BackupMethod): if not os.path.isdir(self.repo): raise YunohostError("backup_no_uncompress_archive_dir") - filesystem.mkdir(self.work_dir, parent=True) + mkdir(self.work_dir, parent=True) ret = subprocess.call(["mount", "-r", "--rbind", self.repo, self.work_dir]) if ret == 0: return @@ -1944,7 +1947,7 @@ class TarBackupMethod(BackupMethod): """ if not os.path.exists(self.repo): - filesystem.mkdir(self.repo, 0o750, parents=True, uid="admin") + mkdir(self.repo, 0o750, parents=True, uid="admin") # Check free space in output self._check_is_enough_free_space() @@ -2667,32 +2670,3 @@ def _recursive_umount(directory): continue return everything_went_fine - - -def disk_usage(path): - # We don't do this in python with os.stat because we don't want - # to follow symlinks - - du_output = check_output(["du", "-sb", path], shell=False) - return int(du_output.split()[0]) - - -def binary_to_human(n, customary=False): - """ - Convert bytes or bits into human readable format with binary prefix - Keyword argument: - n -- Number to convert - customary -- Use customary symbol instead of IEC standard - """ - symbols = ("Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi") - if customary: - symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y") - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return "%.1f%s" % (value, s) - return "%s" % n - diff --git a/src/yunohost/data_migrations/0015_migrate_to_buster.py b/src/yunohost/data_migrations/0015_migrate_to_buster.py index 4f2d4caf8..e71ee1d3a 100644 --- a/src/yunohost/data_migrations/0015_migrate_to_buster.py +++ b/src/yunohost/data_migrations/0015_migrate_to_buster.py @@ -10,8 +10,8 @@ from moulinette.utils.filesystem import read_file from yunohost.tools import Migration, tools_update, tools_upgrade from yunohost.app import unstable_apps from yunohost.regenconf import manually_modified_files -from yunohost.utils.filesystem import free_space_in_directory -from yunohost.utils.packages import ( +from yunohost.utils.system import ( + free_space_in_directory, get_ynh_package_version, _list_upgradable_apt_packages, ) diff --git a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py index 1ccf5ccc9..67df8d771 100644 --- a/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py +++ b/src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py @@ -5,7 +5,7 @@ from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils.log import getActionLogger from yunohost.tools import Migration -from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory +from yunohost.utils.system import free_space_in_directory, space_used_by_directory logger = getActionLogger("yunohost.migration") diff --git a/src/yunohost/log.py b/src/yunohost/log.py index d73a62cd0..4b064fb9c 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -38,7 +38,7 @@ from io import IOBase from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.packages import get_ynh_package_version +from yunohost.utils.system import get_ynh_package_version from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, read_yaml diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index fb9839814..4e0ae660d 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -48,10 +48,12 @@ from yunohost.domain import domain_add from yunohost.firewall import firewall_upnp from yunohost.service import service_start, service_enable from yunohost.regenconf import regen_conf -from yunohost.utils.packages import ( +from yunohost.utils.system import ( _dump_sources_list, _list_upgradable_apt_packages, ynh_packages_version, + dpkg_is_broken, + dpkg_lock_available, ) from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation, OperationLogger @@ -171,20 +173,6 @@ def _set_hostname(hostname, pretty_hostname=None): logger.debug(out) -def _detect_virt(): - """ - Returns the output of systemd-detect-virt (so e.g. 'none' or 'lxc' or ...) - You can check the man of the command to have a list of possible outputs... - """ - - p = subprocess.Popen( - "systemd-detect-virt".split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - - out, _ = p.communicate() - return out.split()[0] - - @is_unit_operation() def tools_postinstall( operation_logger, @@ -463,13 +451,12 @@ def tools_upgrade( apps -- List of apps to upgrade (or [] to update all apps) system -- True to upgrade system """ - from yunohost.utils import packages - if packages.dpkg_is_broken(): + if dpkg_is_broken(): raise YunohostValidationError("dpkg_is_broken") # Check for obvious conflict with other dpkg/apt commands already running in parallel - if not packages.dpkg_lock_available(): + if not dpkg_lock_available(): raise YunohostValidationError("dpkg_lock_not_available") # Legacy options management (--system, --apps) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index c9f70e152..348aa7c4f 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -40,6 +40,7 @@ from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.service import service_status from yunohost.log import is_unit_operation +from yunohost.utils.system import binary_to_human logger = getActionLogger("yunohost.user") @@ -597,7 +598,7 @@ def user_info(username): if has_value: storage_use = int(has_value.group(1)) - storage_use = _convertSize(storage_use) + storage_use = binary_to_human(storage_use) if is_limited: has_percent = re.search(r"%=(\d+)", cmd_result) @@ -1330,15 +1331,6 @@ def user_ssh_remove_key(username, key): # End SSH subcategory # - -def _convertSize(num, suffix=""): - for unit in ["K", "M", "G", "T", "P", "E", "Z"]: - if abs(num) < 1024.0: - return "%3.1f%s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f%s%s" % (num, "Yi", suffix) - - def _hash_user_password(password): """ This function computes and return a salted hash for the password in input. diff --git a/src/yunohost/utils/filesystem.py b/src/yunohost/utils/filesystem.py deleted file mode 100644 index 04d7d3906..000000000 --- a/src/yunohost/utils/filesystem.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- - -""" License - - Copyright (C) 2018 YUNOHOST.ORG - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program; if not, see http://www.gnu.org/licenses - -""" -import os - - -def free_space_in_directory(dirpath): - stat = os.statvfs(dirpath) - return stat.f_frsize * stat.f_bavail - - -def space_used_by_directory(dirpath): - stat = os.statvfs(dirpath) - return stat.f_frsize * stat.f_blocks diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/system.py similarity index 67% rename from src/yunohost/utils/packages.py rename to src/yunohost/utils/system.py index 3105bc4c7..f7d37cf1c 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/system.py @@ -23,13 +23,85 @@ import os import logging from moulinette.utils.process import check_output -from packaging import version +from yunohost.utils.error import YunohostError logger = logging.getLogger("yunohost.utils.packages") YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat"] +def system_arch(): + return check_output("dpkg --print-architecture") + + +def system_virt(): + """ + Returns the output of systemd-detect-virt (so e.g. 'none' or 'lxc' or ...) + You can check the man of the command to have a list of possible outputs... + """ + # Detect virt technology (if not bare metal) and arch + # Gotta have this "|| true" because it systemd-detect-virt return 'none' + # with an error code on bare metal ~.~ + return check_output("systemd-detect-virt || true") + + +def free_space_in_directory(dirpath): + stat = os.statvfs(dirpath) + return stat.f_frsize * stat.f_bavail + + +def space_used_by_directory(dirpath, follow_symlinks=True): + + if not follow_symlinks: + du_output = check_output(["du", "-sb", dirpath], shell=False) + return int(du_output.split()[0]) + + stat = os.statvfs(dirpath) + return stat.f_frsize * stat.f_blocks + + +def human_to_binary(size: str) -> int: + + symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y") + factor = {} + for i, s in enumerate(symbols): + factor[s] = 1 << (i + 1) * 10 + + suffix = size[-1] + size = size[:-1] + + if suffix not in symbols: + raise YunohostError(f"Invalid size suffix '{suffix}', expected one of {symbols}") + + try: + size = float(size) + except Exception: + raise YunohostError(f"Failed to convert size {size} to float") + + return size * factor[suffix] + + +def binary_to_human(n: int) -> str: + """ + Convert bytes or bits into human readable format with binary prefix + """ + symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y") + prefix = {} + for i, s in enumerate(symbols): + prefix[s] = 1 << (i + 1) * 10 + for s in reversed(symbols): + if n >= prefix[s]: + value = float(n) / prefix[s] + return "%.1f%s" % (value, s) + return "%s" % n + + +def ram_available(): + + import psutil + return (psutil.virtual_memory().available, psutil.swap_memory().free) + + def get_ynh_package_version(package): # Returns the installed version and release version ('stable' or 'testing' @@ -48,43 +120,6 @@ def get_ynh_package_version(package): return {"version": out[1].strip("()"), "repo": out[2].strip(";")} -def meets_version_specifier(pkg_name, specifier): - """ - Check if a package installed version meets specifier - - specifier is something like ">> 1.2.3" - """ - - # In practice, this function is only used to check the yunohost version - # installed. - # We'll trim any ~foobar in the current installed version because it's not - # handled correctly by version.parse, but we don't care so much in that - # context - assert pkg_name in YUNOHOST_PACKAGES - pkg_version = get_ynh_package_version(pkg_name)["version"] - pkg_version = re.split(r"\~|\+|\-", pkg_version)[0] - pkg_version = version.parse(pkg_version) - - # Extract operator and version specifier - op, req_version = re.search(r"(<<|<=|=|>=|>>) *([\d\.]+)", specifier).groups() - req_version = version.parse(req_version) - - # Python2 had a builtin that returns (-1, 0, 1) depending on comparison - # c.f. https://stackoverflow.com/a/22490617 - def cmp(a, b): - return (a > b) - (a < b) - - deb_operators = { - "<<": lambda v1, v2: cmp(v1, v2) in [-1], - "<=": lambda v1, v2: cmp(v1, v2) in [-1, 0], - "=": lambda v1, v2: cmp(v1, v2) in [0], - ">=": lambda v1, v2: cmp(v1, v2) in [0, 1], - ">>": lambda v1, v2: cmp(v1, v2) in [1], - } - - return deb_operators[op](pkg_version, req_version) - - def ynh_packages_version(*args, **kwargs): # from cli the received arguments are: # (Namespace(_callbacks=deque([]), _tid='_global', _to_return={}), []) {} From 744729713d469f3ef94a9a40e9aa5013097f3c84 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Nov 2021 22:06:32 +0100 Subject: [PATCH 12/54] Add notes for app resources logic --- src/yunohost/utils/resources.py | 129 +++++++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 3 deletions(-) diff --git a/src/yunohost/utils/resources.py b/src/yunohost/utils/resources.py index 424e38f91..64ec67b8c 100644 --- a/src/yunohost/utils/resources.py +++ b/src/yunohost/utils/resources.py @@ -59,6 +59,21 @@ class AppResourceSet: class AptDependenciesAppResource(AppResource): + """ + is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn) + is_available -> True? idk + + update -> update deps on __APP__-ynh-deps + provision -> create/update deps on __APP__-ynh-deps + + deprovision -> remove __APP__-ynh-deps (+autoremove?) + + deep_clean -> remove any __APP__-ynh-deps for app not in app list + + backup -> nothing + restore = provision + """ + type = "apt" default_properties = { @@ -71,7 +86,23 @@ class AptDependenciesAppResource(AppResource): # call helpers idk ... pass + class SourcesAppResource(AppResource): + """ + is_provisioned -> (if pre_download,) cache exists with appropriate checksum + is_available -> curl HEAD returns 200 + + update -> none? + provision -> full download + check checksum + + deprovision -> remove cache for __APP__ ? + + deep_clean -> remove all cache + + backup -> nothing + restore -> nothing + """ + type = "sources" default_properties = { @@ -85,6 +116,21 @@ class SourcesAppResource(AppResource): class RoutesAppResource(AppResource): + """ + is_provisioned -> main perm exists + is_available -> perm urls do not conflict + + update -> refresh/update values for url/additional_urls/show_tile/auth/protected/... create new perms / delete any perm not listed + provision -> same as update? + + deprovision -> delete permissions + + deep_clean -> delete permissions for any __APP__.foobar where app not in app list... + + backup -> handled elsewhere by the core, should be integrated in there (dump .ldif/yml?) + restore -> handled by the core, should be integrated in there (restore .ldif/yml?) + """ + type = "routes" default_properties = { @@ -136,10 +182,26 @@ class RoutesAppResource(AppResource): class PortAppResource(AppResource): + """ + is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting) + is_available -> true + + update -> true + provision -> find a port not used by any app + + deprovision -> delete the port setting + + deep_clean -> ? + + backup -> nothing (backup port setting) + restore -> nothing (restore port setting) + """ + type = "port" default_properties = { - "value": 1000 + "value": 1000, + "type": "internal", } def _port_is_used(self, port): @@ -167,11 +229,26 @@ class PortAppResource(AppResource): class SystemuserAppResource(AppResource): + """ + is_provisioned -> user __APP__ exists + is_available -> user and group __APP__ doesn't exists + + update -> update values for home / shell / groups + provision -> create user + + deprovision -> delete user + + deep_clean -> uuuuh ? delete any user that could correspond to an app x_x ? + + backup -> nothing + restore -> provision + """ + type = "system_user" default_properties = { "username": "__APP__", - "home_dir": "/var/www/__APP__", + "home_dir": "__INSTALL_DIR__", "use_shell": False, "groups": [] } @@ -190,11 +267,27 @@ class SystemuserAppResource(AppResource): class InstalldirAppResource(AppResource): + """ + is_provisioned -> setting install_dir exists + /dir/ exists + is_available -> /dir/ doesn't exists + + provision -> create setting + create dir + update -> update perms ? + + deprovision -> delete dir + delete setting + + deep_clean -> uuuuh ? delete any dir in /var/www/ that would not correspond to an app x_x ? + + backup -> cp install dir + restore -> cp install dir + """ + type = "install_dir" default_properties = { "dir": "/var/www/__APP__", # FIXME or choose to move this elsewhere nowadays idk... - "alias": "final_path" + "alias": "final_path", + # FIXME : add something about perms ? } # FIXME: change default dir to /opt/stuff if app ain't a webapp ... @@ -219,6 +312,21 @@ class InstalldirAppResource(AppResource): class DatadirAppResource(AppResource): + """ + is_provisioned -> setting data_dir exists + /dir/ exists + is_available -> /dir/ doesn't exists + + provision -> create setting + create dir + update -> update perms ? + + deprovision -> (only if purge enabled...) delete dir + delete setting + + deep_clean -> zblerg idk nothing + + backup -> cp data dir ? (if not backup_core_only) + restore -> cp data dir ? (if in backup) + """ + type = "data_dir" default_properties = { @@ -239,6 +347,21 @@ class DatadirAppResource(AppResource): class DBAppResource(AppResource): + """ + is_provisioned -> setting db_user, db_name, db_pwd exists + is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it) + + provision -> setup the db + init the setting + update -> ?? + + deprovision -> delete the db + + deep_clean -> ... idk look into any db name that would not be related to any app ... + + backup -> dump db + restore -> setup + inject db dump + """ + type = "db" default_properties = { From 7206be00201799bd04af052a0dc0a6c94a7616fd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Nov 2021 17:09:30 +0100 Subject: [PATCH 13/54] Tmp work on provision/deprovision apt and system user --- src/yunohost/app.py | 107 ++++--- src/yunohost/utils/resources.py | 542 ++++++++++++++++++++------------ 2 files changed, 398 insertions(+), 251 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e251b99de..409e94d2d 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -735,6 +735,37 @@ def app_manifest(app): return manifest +def _confirm_app_install(app, force=False): + + # Ignore if there's nothing for confirm (good quality app), if --force is used + # or if request on the API (confirm already implemented on the API side) + if force or Moulinette.interface.type == "api": + return + + quality = _app_quality(app) + if quality == "success": + return + + # i18n: confirm_app_install_warning + # i18n: confirm_app_install_danger + # i18n: confirm_app_install_thirdparty + + if quality in ["danger", "thirdparty"]: + answer = Moulinette.prompt( + m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"), + color="red", + ) + if answer != "Yes, I understand": + raise YunohostError("aborting") + + else: + answer = Moulinette.prompt( + m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow" + ) + if answer.upper() != "Y": + raise YunohostError("aborting") + + @is_unit_operation() def app_install( operation_logger, @@ -776,37 +807,7 @@ def app_install( if free_space_in_directory("/") <= 512 * 1000 * 1000: raise YunohostValidationError("disk_space_not_sufficient_install") - def confirm_install(app): - - # Ignore if there's nothing for confirm (good quality app), if --force is used - # or if request on the API (confirm already implemented on the API side) - if force or Moulinette.interface.type == "api": - return - - quality = _app_quality(app) - if quality == "success": - return - - # i18n: confirm_app_install_warning - # i18n: confirm_app_install_danger - # i18n: confirm_app_install_thirdparty - - if quality in ["danger", "thirdparty"]: - answer = Moulinette.prompt( - m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"), - color="red", - ) - if answer != "Yes, I understand": - raise YunohostError("aborting") - - else: - answer = Moulinette.prompt( - m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow" - ) - if answer.upper() != "Y": - raise YunohostError("aborting") - - confirm_install(app) + _confirm_app_install(app) manifest, extracted_app_folder = _extract_app(app) packaging_format = manifest["packaging_format"] @@ -815,7 +816,6 @@ def app_install( raise YunohostValidationError("app_id_invalid") app_id = manifest["id"] - label = label if label else manifest["name"] # Check requirements _check_manifest_requirements(manifest, action="install") @@ -829,6 +829,8 @@ def app_install( else: app_instance_name = app_id + app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) + # Retrieve arguments list for install script raw_questions = manifest["install"] questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args) @@ -861,7 +863,6 @@ def app_install( logger.info(m18n.n("app_start_install", app=app_id)) # Create app directory - app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) if os.path.exists(app_setting_path): shutil.rmtree(app_setting_path) os.makedirs(app_setting_path) @@ -894,23 +895,31 @@ def app_install( recursive=True, ) + # Initialize the main permission for the app + # The permission is initialized with no url associated, and with tile disabled + # For web app, the root path of the app will be added as url and the tile + # will be enabled during the app install. C.f. 'app_register_url()' below. + if packaging_format >= 2: + init_main_perm_allowed = ["visitors"] if not args.get("is_public") else ["all_users"] + else: + init_main_perm_allowed = ["all_users"] - resources = AppResourceSet(manifest["resources"], app_instance_name) - resources.check_availability() - resources.provision() + permission_create( + app_instance_name + ".main", + allowed=init_main_perm_allowed, + label=label if label else manifest["name"], + show_tile=False, + protected=False, + ) - if packaging_format < 2: - # Initialize the main permission for the app - # The permission is initialized with no url associated, and with tile disabled - # For web app, the root path of the app will be added as url and the tile - # will be enabled during the app install. C.f. 'app_register_url()' below. - permission_create( - app_instance_name + ".main", - allowed=["all_users"], - label=label, - show_tile=False, - protected=False, - ) + if packaging_format >= 2: + try: + from yunohost.utils.resources import AppResourceManager + resources = AppResourceManager(app_instance_name, current=app_setting_path, wanted=extracted_app_folder) + resources.apply() + except: + raise + # FIXME : error handling # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( @@ -2519,7 +2528,7 @@ def _make_environment_for_app_script( # If packaging format v2, load all settings if manifest["packaging_format"] >= 2: env_dict["app"] = app - for setting_name, setting_value in _get_app_settings(app): + for setting_name, setting_value in _get_app_settings(app).items(): # Ignore special internal settings like checksum__ # (not a huge deal to load them but idk...) diff --git a/src/yunohost/utils/resources.py b/src/yunohost/utils/resources.py index 64ec67b8c..c9969f01b 100644 --- a/src/yunohost/utils/resources.py +++ b/src/yunohost/utils/resources.py @@ -22,14 +22,47 @@ import os import copy from typing import Dict, Any +from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file + from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.hook import hook_exec + +logger = getActionLogger("yunohost.app_resources") + + +class AppResourceManager(object): + + def __init__(self, app: str, manifest: str): + + self.app = app + self.resources = {name: AppResourceClassesByType[name](infos, app) + for name, infos in resources_dict.items()} + + def apply(self): + + + + + + + def validate_resource_availability(self): + + for name, resource in self.resources.items(): + resource.validate_availability(context={}) + + def provision_or_update_resources(self): + + for name, resource in self.resources.items(): + logger.info("Running provision_or_upgrade for {self.type}") + resource.provision_or_update(context={}) class AppResource(object): - def __init__(self, properties: Dict[str, Any], app_id: str): + def __init__(self, properties: Dict[str, Any], app: str): - self.app_id = app_id + self.app = app for key, value in self.default_properties.items(): setattr(self, key, value) @@ -37,85 +70,48 @@ class AppResource(object): for key, value in properties.items(): setattr(self, key, value) - def get_app_settings(self): - from yunohost.app import _get_app_settings - return _get_app_settings(self.app_id) + def get_setting(self, key): + from yunohost.app import app_setting + return app_setting(self.app, key) - def check_availability(self, context: Dict): + def set_setting(self, key, value): + from yunohost.app import app_setting + app_setting(self.app, key, value=value) + + def delete_setting(self, key, value): + from yunohost.app import app_setting + app_setting(self.app, key, delete=True) + + def validate_availability(self, context: Dict): pass + def _run_script(self, action, script, env={}, user="root"): -class AppResourceSet: + from yunohost.app import _make_tmp_workdir_for_app, _make_environment_for_app_script - def __init__(self, resources_dict: Dict[str, Dict[str, Any]], app_id: str): + tmpdir = _make_tmp_workdir_for_app(app=self.app) - self.set = {name: AppResourceClassesByType[name](infos, app_id) - for name, infos in resources_dict.items()} + env_ = _make_environment_for_app_script(self.app, workdir=tmpdir, action=f"{action}_{self.type}") + env_.update(env) - def check_availability(self): + script_path = f"{tmpdir}/{action}_{self.type}" + script = f""" +source /usr/share/yunohost/helpers +ynh_abort_if_errors - for name, resource in self.set.items(): - resource.check_availability(context={}) +{script} +""" + + write_to_file(script_path, script) + + print(env_) + + # FIXME : use the hook_exec_with_debug_instructions_stuff + ret, _ = hook_exec(script_path, env=env_) + print(ret) -class AptDependenciesAppResource(AppResource): - """ - is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn) - is_available -> True? idk - - update -> update deps on __APP__-ynh-deps - provision -> create/update deps on __APP__-ynh-deps - - deprovision -> remove __APP__-ynh-deps (+autoremove?) - - deep_clean -> remove any __APP__-ynh-deps for app not in app list - - backup -> nothing - restore = provision - """ - - type = "apt" - - default_properties = { - "packages": [], - "extras": {} - } - - def check_availability(self, context): - # ? FIXME - # call helpers idk ... - pass - - -class SourcesAppResource(AppResource): - """ - is_provisioned -> (if pre_download,) cache exists with appropriate checksum - is_available -> curl HEAD returns 200 - - update -> none? - provision -> full download + check checksum - - deprovision -> remove cache for __APP__ ? - - deep_clean -> remove all cache - - backup -> nothing - restore -> nothing - """ - - type = "sources" - - default_properties = { - "main": {"url": "?", "sha256sum": "?", "predownload": True} - } - - def check_availability(self, context): - # ? FIXME - # call request.head on the url idk - pass - - -class RoutesAppResource(AppResource): +class WebpathResource(AppResource): """ is_provisioned -> main perm exists is_available -> perm urls do not conflict @@ -131,101 +127,30 @@ class RoutesAppResource(AppResource): restore -> handled by the core, should be integrated in there (restore .ldif/yml?) """ - type = "routes" + type = "webpath" + priority = 10 default_properties = { "full_domain": False, - "main": { - "url": "/", - "additional_urls": [], - "init_allowed": "__FIXME__", - "show_tile": True, - "protected": False, - "auth_header": True, - "label": "FIXME", - } } - def check_availability(self, context): + def validate_availability(self, context): from yunohost.app import _assert_no_conflicting_apps - app_settings = self.get_app_settings() - domain = app_settings["domain"] - path = app_settings["path"] if not self.full_domain else "/" - _assert_no_conflicting_apps(domain, path, ignore_app=self.app_id) + domain = self.get_setting("domain") + path = self.get_setting("path") if not self.full_domain else "/" + _assert_no_conflicting_apps(domain, path, ignore_app=self.app) def provision_or_update(self, context: Dict): - if context["app_action"] == "install": - pass # FIXME - # Initially, the .main permission is created with no url at all associated - # When the app register/books its web url, we also add the url '/' - # (meaning the root of the app, domain.tld/path/) - # and enable the tile to the SSO, and both of this should match 95% of apps - # For more specific cases, the app is free to change / add urls or disable - # the tile using the permission helpers. - #permission_create( - # self.app_id + ".main", - # allowed=["all_users"], - # label=label, - # show_tile=False, - # protected=False, - #) - #permission_url(app + ".main", url="/", sync_perm=False) - #user_permission_update(app + ".main", show_tile=True, sync_perm=False) - #permission_sync_to_user() + # Nothing to do ? Just setting the domain/path during install + # already provisions it ... + return # FIXME def deprovision(self, context: Dict): - del context["app_settings"]["domain"] - del context["app_settings"]["path"] - - -class PortAppResource(AppResource): - """ - is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting) - is_available -> true - - update -> true - provision -> find a port not used by any app - - deprovision -> delete the port setting - - deep_clean -> ? - - backup -> nothing (backup port setting) - restore -> nothing (restore port setting) - """ - - type = "port" - - default_properties = { - "value": 1000, - "type": "internal", - } - - def _port_is_used(self, port): - - # FIXME : this could be less brutal than two os.system ... - cmd1 = "ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ':%s$'" % port - # This second command is mean to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up) - cmd2 = f"grep -q \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml" - return os.system(cmd1) == 0 and os.system(cmd2) == 0 - - def provision_or_update(self, context: str): - - # Don't do anything if port already defined ? - if context["app_settings"].get("port"): - return - - port = self.value - while self._port_is_used(port): - port += 1 - - context["app_settings"]["port"] = port - - def deprovision(self, context: Dict): - raise NotImplementedError() + self.delete_setting("domain") + self.delete_setting("path") class SystemuserAppResource(AppResource): @@ -233,8 +158,8 @@ class SystemuserAppResource(AppResource): is_provisioned -> user __APP__ exists is_available -> user and group __APP__ doesn't exists - update -> update values for home / shell / groups provision -> create user + update -> update values for home / shell / groups deprovision -> delete user @@ -245,26 +170,69 @@ class SystemuserAppResource(AppResource): """ type = "system_user" + priority = 20 default_properties = { - "username": "__APP__", - "home_dir": "__INSTALL_DIR__", - "use_shell": False, - "groups": [] + "allow_ssh": [] + "allow_sftp": [] } - def check_availability(self, context): - if os.system(f"getent passwd {self.username} &>/dev/null") != 0: - raise YunohostValidationError(f"User {self.username} already exists") - if os.system(f"getent group {self.username} &>/dev/null") != 0: - raise YunohostValidationError(f"Group {self.username} already exists") + def validate_availability(self, context): + pass + # FIXME : do we care if user already exists ? shouldnt we assume that user $app corresponds to the app ...? + + # FIXME : but maybe we should at least check that no corresponding yunohost user exists + + #if os.system(f"getent passwd {self.username} &>/dev/null") != 0: + # raise YunohostValidationError(f"User {self.username} already exists") + #if os.system(f"getent group {self.username} &>/dev/null") != 0: + # raise YunohostValidationError(f"Group {self.username} already exists") + + def provision_or_update(self, context: Dict): + + if os.system(f"getent passwd {self.app} &>/dev/null") != 0: + cmd = f"useradd --system --user-group {self.app}" + os.system(cmd) + + if os.system(f"getent passwd {self.app} &>/dev/null") == 0: + raise YunohostError(f"Failed to create system user for {self.app}") + + groups = [] + if self.allow_ssh: + groups.append("ssh.app") + if self.allow_sftp: + groups.append("sftp.app") + groups = + + cmd = f"usermod -a -G {groups} {self.app}" + # FIXME : handle case where group gets removed + os.system(cmd) + +# useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account" +# for group in $groups; do +# usermod -a -G "$group" "$username" +# done + + +# | arg: -g, --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp) - def provision_or_update(self, context: str): - raise NotImplementedError() def deprovision(self, context: Dict): - raise NotImplementedError() + self._run_script("deprovision", + f'ynh_system_user_delete "{self.username}"') + +# # Check if the user exists on the system +#if os.system(f"getent passwd {self.username} &>/dev/null") != 0: +# if ynh_system_user_exists "$username"; then +# deluser $username +# fi +# # Check if the group exists on the system +#if os.system(f"getent group {self.username} &>/dev/null") != 0: +# if ynh_system_group_exists "$username"; then +# delgroup $username +# fi +# class InstalldirAppResource(AppResource): """ @@ -283,32 +251,54 @@ class InstalldirAppResource(AppResource): """ type = "install_dir" + priority = 30 default_properties = { "dir": "/var/www/__APP__", # FIXME or choose to move this elsewhere nowadays idk... - "alias": "final_path", - # FIXME : add something about perms ? + "alias": None, + "owner": "__APP__:rx", + "group": "__APP__:rx", } # FIXME: change default dir to /opt/stuff if app ain't a webapp ... + # FIXME: what do in a scenario where the location changed - def check_availability(self, context): - if os.path.exists(self.dir): - raise YunohostValidationError(f"Folder {self.dir} already exists") + def validate_availability(self, context): + pass def provision_or_update(self, context: Dict): - if context["app_action"] in ["install", "restore"]: - if os.path.exists(self.dir): - raise YunohostValidationError(f"Path {self.dir} already exists") + current_install_dir = self.get_setting("install_dir") - if "installdir" not in context["app_settings"]: - context["app_settings"]["installdir"] = self.dir - context["app_settings"][self.alias] = context["app_settings"]["installdir"] + # If during install, /var/www/$app already exists, assume that it's okay to remove and recreate it + # FIXME : is this the right thing to do ? + if not current_install_dir and os.path.isdir(self.dir): + rm(self.dir, recursive=True) + + if not os.path.isdir(self.dir): + # Handle case where install location changed, in which case we shall move the existing install dir + if current_install_dir and os.path.isdir(current_install_dir): + shutil.move(current_install_dir, self.dir) + else: + mkdir(self.dir) + + owner, owner_perm = self.owner.split(":") + group, group_perm = self.group.split(":") + owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0) + group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0) + perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0" + + chmod(self.dir, oct(int(perm_octal))) + chown(self.dir, owner, group) + + self.set_setting("install_dir", self.dir) + if self.alias: + self.set_setting(self.alias, self.dir) def deprovision(self, context: Dict): - # FIXME: should it rm the directory during remove/deprovision ? - pass + # FIXME : check that self.dir has a sensible value to prevent catastrophes + if os.path.isdir(self.dir): + rm(self.dir, recursive=True) class DatadirAppResource(AppResource): @@ -328,56 +318,204 @@ class DatadirAppResource(AppResource): """ type = "data_dir" + priority = 40 default_properties = { "dir": "/home/yunohost.app/__APP__", # FIXME or choose to move this elsewhere nowadays idk... + "owner": "__APP__:rx", + "group": "__APP__:rx", } - def check_availability(self, context): - if os.path.exists(self.dir): - raise YunohostValidationError(f"Folder {self.dir} already exists") + def validate_availability(self, context): + pass + # Nothing to do ? If datadir already exists then it may be legit data + # from a previous install def provision_or_update(self, context: Dict): - if "datadir" not in context["app_settings"]: - context["app_settings"]["datadir"] = self.dir + + current_data_dir = self.get_setting("data_dir") + + if not os.path.isdir(self.dir): + # Handle case where install location changed, in which case we shall move the existing install dir + if current_data_dir and os.path.isdir(current_data_dir): + shutil.move(current_data_dir, self.dir) + else: + mkdir(self.dir) + + owner, owner_perm = self.owner.split(":") + group, group_perm = self.group.split(":") + owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0) + group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0) + perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0" + + chmod(self.dir, oct(int(perm_octal))) + chown(self.dir, owner, group) + + self.set_setting("data_dir", self.dir) def deprovision(self, context: Dict): - # FIXME: should it rm the directory during remove/deprovision ? + # FIXME: This should rm the datadir only if purge is enabled pass + #if os.path.isdir(self.dir): + # rm(self.dir, recursive=True) -class DBAppResource(AppResource): +# +#class SourcesAppResource(AppResource): +# """ +# is_provisioned -> (if pre_download,) cache exists with appropriate checksum +# is_available -> curl HEAD returns 200 +# +# update -> none? +# provision -> full download + check checksum +# +# deprovision -> remove cache for __APP__ ? +# +# deep_clean -> remove all cache +# +# backup -> nothing +# restore -> nothing +# """ +# +# type = "sources" +# +# default_properties = { +# "main": {"url": "?", "sha256sum": "?", "predownload": True} +# } +# +# def validate_availability(self, context): +# # ? FIXME +# # call request.head on the url idk +# pass +# +# def provision_or_update(self, context: Dict): +# # FIXME +# return +# + +class AptDependenciesAppResource(AppResource): """ - is_provisioned -> setting db_user, db_name, db_pwd exists - is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it) + is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn) + is_available -> True? idk - provision -> setup the db + init the setting - update -> ?? + update -> update deps on __APP__-ynh-deps + provision -> create/update deps on __APP__-ynh-deps - deprovision -> delete the db + deprovision -> remove __APP__-ynh-deps (+autoremove?) - deep_clean -> ... idk look into any db name that would not be related to any app ... + deep_clean -> remove any __APP__-ynh-deps for app not in app list - backup -> dump db - restore -> setup + inject db dump + backup -> nothing + restore = provision """ - type = "db" + type = "apt" + priority = 50 default_properties = { - "type": "mysql" + "packages": [], + "extras": {} } - def check_availability(self, context): - # FIXME : checking availability sort of imply that mysql / postgresql is installed - # or we gotta make sure mariadb-server or postgresql is gonna be installed (apt resource) + def validate_availability(self, context): + # ? FIXME + # call helpers idk ... pass - def provision_or_update(self, context: str): - raise NotImplementedError() + def provision_or_update(self, context: Dict): + + # FIXME : implement 'extras' management + self._run_script("provision_or_update", + "ynh_install_app_dependencies $apt_dependencies", + {"apt_dependencies": self.packages}) def deprovision(self, context: Dict): - raise NotImplementedError() + self._run_script("deprovision", + "ynh_remove_app_dependencies") + + +class PortAppResource(AppResource): + """ + is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting) + is_available -> true + + update -> true + provision -> find a port not used by any app + + deprovision -> delete the port setting + + deep_clean -> ? + + backup -> nothing (backup port setting) + restore -> nothing (restore port setting) + """ + + type = "port" + priority = 70 + + default_properties = { + "default": 1000, + "type": "internal", # FIXME : implement logic for exposed port (allow/disallow in firewall ?) + } + + def _port_is_used(self, port): + + # FIXME : this could be less brutal than two os.system ... + cmd1 = "ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ':%s$'" % port + # This second command is mean to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up) + cmd2 = f"grep --quiet \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml" + return os.system(cmd1) == 0 and os.system(cmd2) == 0 + + def provision_or_update(self, context: str): + + # Don't do anything if port already defined ? + if self.get_setting("port"): + return + + port = self.default + while self._port_is_used(port): + port += 1 + + self.set_setting("port", port) + + def deprovision(self, context: Dict): + + self.delete_setting("port") + + +#class DBAppResource(AppResource): +# """ +# is_provisioned -> setting db_user, db_name, db_pwd exists +# is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it) +# +# provision -> setup the db + init the setting +# update -> ?? +# +# deprovision -> delete the db +# +# deep_clean -> ... idk look into any db name that would not be related to any app ... +# +# backup -> dump db +# restore -> setup + inject db dump +# """ +# +# type = "db" +# +# default_properties = { +# "type": "mysql" +# } +# +# def validate_availability(self, context): +# # FIXME : checking availability sort of imply that mysql / postgresql is installed +# # or we gotta make sure mariadb-server or postgresql is gonna be installed (apt resource) +# pass +# +# def provision_or_update(self, context: str): +# raise NotImplementedError() +# +# def deprovision(self, context: Dict): +# raise NotImplementedError() +# AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()} From 364a3bc70ab82be0c3d02d4f546df8e191eed30f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Dec 2021 19:02:19 +0100 Subject: [PATCH 14/54] manifestv2: fix many things, have resource system somewhat working for install/remove --- helpers/apt | 3 +- helpers/utils | 23 +++++-- locales/en.json | 2 +- src/app.py | 58 ++++++++++++------ src/utils/resources.py | 134 +++++++++++++++++++++++++---------------- 5 files changed, 140 insertions(+), 80 deletions(-) diff --git a/helpers/apt b/helpers/apt index 74bec758c..433318634 100644 --- a/helpers/apt +++ b/helpers/apt @@ -227,9 +227,8 @@ ynh_install_app_dependencies() { # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" local dependencies=${dependencies//|/ | } - local manifest_path="$YNH_APP_BASEDIR/manifest.json" - local version=$(jq -r '.version' "$manifest_path") + local version=$(ynh_read_manifest --manifest_key="version") if [ -z "${version}" ] || [ "$version" == "null" ]; then version="1.0" fi diff --git a/helpers/utils b/helpers/utils index 16e353dc4..a2484bb8b 100644 --- a/helpers/utils +++ b/helpers/utils @@ -765,12 +765,25 @@ ynh_read_manifest() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if [ ! -e "$manifest" ]; then + if [ ! -e "${manifest:-}" ]; then # If the manifest isn't found, try the common place for backup and restore script. - manifest="$YNH_APP_BASEDIR/manifest.json" + if [ -e "$YNH_APP_BASEDIR/manifest.json" ] + then + manifest="$YNH_APP_BASEDIR/manifest.json" + elif [ -e "$YNH_APP_BASEDIR/manifest.toml" ] + then + manifest="$YNH_APP_BASEDIR/manifest.toml" + else + ynh_die --message "No manifest found !?" + fi fi - jq ".$manifest_key" "$manifest" --raw-output + if echo "$manifest" | grep -q '\.json$' + then + jq ".$manifest_key" "$manifest" --raw-output + else + cat "$manifest" | python3 -c 'import json, toml, sys; print(json.dumps(toml.load(sys.stdin)))' | jq ".$manifest_key" --raw-output + fi } # Read the upstream version from the manifest or `$YNH_APP_MANIFEST_VERSION` @@ -914,9 +927,7 @@ ynh_compare_current_package_version() { _ynh_apply_default_permissions() { local target=$1 - local ynh_requirement=$(jq -r '.requirements.yunohost' $YNH_APP_BASEDIR/manifest.json | tr -d '>= ') - - if [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then chmod o-rwx $target chmod g-w $target chown -R root:root $target diff --git a/locales/en.json b/locales/en.json index fc4e4283a..cc5ad956a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -43,7 +43,7 @@ "app_packaging_format_not_supported": "This app cannot be installed because its packaging format is not supported by your YunoHost version. You should probably consider upgrading your system.", "app_remove_after_failed_install": "Removing the app following the installation failure...", "app_removed": "{app} uninstalled", - "app_requirements_checking": "Checking required packages for {app}...", + "app_requirements_checking": "Checking requirements for {app}...", "app_requirements_unmeet": "Requirements are not met for {app}, the package {pkgname} ({version}) must be {spec}", "app_restore_failed": "Could not restore {app}: {error}", "app_restore_script_failed": "An error occured inside the app restore script", diff --git a/src/app.py b/src/app.py index a174d3eca..d2aa0fa65 100644 --- a/src/app.py +++ b/src/app.py @@ -59,7 +59,6 @@ from yunohost.utils.config import ( DomainQuestion, PathQuestion, ) -from yunohost.utils.resources import AppResourceSet from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.system import ( @@ -790,7 +789,7 @@ def app_install( if free_space_in_directory("/") <= 512 * 1000 * 1000: raise YunohostValidationError("disk_space_not_sufficient_install") - _confirm_app_install(app) + _confirm_app_install(app, force) manifest, extracted_app_folder = _extract_app(app) packaging_format = manifest["packaging_format"] @@ -824,10 +823,11 @@ def app_install( } # Validate domain / path availability for webapps - if packaging_format < 2: - path_requirement = _guess_webapp_path_requirement(extracted_app_folder) - _validate_webpath_requirement(args, path_requirement) + # (ideally this should be handled by the resource system for manifest v >= 2 + path_requirement = _guess_webapp_path_requirement(extracted_app_folder) + _validate_webpath_requirement(args, path_requirement) + if packaging_format < 2: # Attempt to patch legacy helpers ... _patch_legacy_helpers(extracted_app_folder) @@ -881,9 +881,14 @@ def app_install( # Initialize the main permission for the app # The permission is initialized with no url associated, and with tile disabled # For web app, the root path of the app will be added as url and the tile - # will be enabled during the app install. C.f. 'app_register_url()' below. + # will be enabled during the app install. C.f. 'app_register_url()' below + # or the webpath resource if packaging_format >= 2: - init_main_perm_allowed = ["visitors"] if not args.get("is_public") else ["all_users"] + if args.get("init_permission_main"): + init_main_perm_allowed = args.get("init_permission_main") + else: + init_main_perm_allowed = ["visitors"] if not args.get("is_public") else ["all_users"] + else: init_main_perm_allowed = ["all_users"] @@ -896,13 +901,13 @@ def app_install( ) if packaging_format >= 2: + from yunohost.utils.resources import AppResourceManager try: - from yunohost.utils.resources import AppResourceManager - resources = AppResourceManager(app_instance_name, current=app_setting_path, wanted=extracted_app_folder) - resources.apply() - except: + AppResourceManager(app_instance_name, wanted=manifest["resources"], current={}).apply() + except Exception: + # FIXME : improve error handling .... + AppResourceManager(app_instance_name, wanted={}, current=manifest["resources"]).apply() raise - # FIXME : error handling # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( @@ -1000,6 +1005,14 @@ def app_install( m18n.n("unexpected_error", error="\n" + traceback.format_exc()) ) + if packaging_format >= 2: + from yunohost.utils.resources import AppResourceManager + try: + AppResourceManager(app_instance_name, wanted={}, current=manifest["resources"]).apply() + except Exception: + # FIXME : improve error handling .... + raise + # Remove all permission in LDAP for permission_name in user_permission_list()["permissions"].keys(): if permission_name.startswith(app_instance_name + "."): @@ -1103,21 +1116,30 @@ def app_remove(operation_logger, app, purge=False): finally: shutil.rmtree(tmp_workdir_for_app) - if ret == 0: - logger.success(m18n.n("app_removed", app=app)) - hook_callback("post_app_remove", env=env_dict) - else: - logger.warning(m18n.n("app_not_properly_removed", app=app)) - # Remove all permission in LDAP for permission_name in user_permission_list(apps=[app])["permissions"].keys(): permission_delete(permission_name, force=True, sync_perm=False) + packaging_format = manifest["packaging_format"] + if packaging_format >= 2: + try: + from yunohost.utils.resources import AppResourceManager + AppResourceManager(app, wanted={}, current=manifest["resources"]).apply() + except Exception: + # FIXME : improve error handling .... + raise + if os.path.exists(app_setting_path): shutil.rmtree(app_setting_path) hook_remove(app) + if ret == 0: + logger.success(m18n.n("app_removed", app=app)) + hook_callback("post_app_remove", env=env_dict) + else: + logger.warning(m18n.n("app_not_properly_removed", app=app)) + permission_sync_to_user() _assert_system_is_sane_for_app(manifest, "post") diff --git a/src/utils/resources.py b/src/utils/resources.py index c9969f01b..b0956a2d0 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -20,10 +20,15 @@ """ import os import copy +import shutil from typing import Dict, Any +from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file +from moulinette.utils.filesystem import ( + rm, +) from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.hook import hook_exec @@ -31,43 +36,57 @@ from yunohost.hook import hook_exec logger = getActionLogger("yunohost.app_resources") -class AppResourceManager(object): +class AppResourceManager: - def __init__(self, app: str, manifest: str): + def __init__(self, app: str, current: Dict, wanted: Dict): self.app = app - self.resources = {name: AppResourceClassesByType[name](infos, app) - for name, infos in resources_dict.items()} + self.current = current + self.wanted = wanted - def apply(self): + def apply(self, **context): + + for name, infos in self.wanted.items(): + resource = AppResourceClassesByType[name](infos, self.app) + # FIXME: not a great place to check this because here + # we already started an operation + # We should find a way to validate this before actually starting + # the install procedure / theoperation log + if name not in self.current.keys(): + resource.validate_availability(context=context) + + for name, infos in reversed(self.current.items()): + if name not in self.wanted.keys(): + resource = AppResourceClassesByType[name](infos, self.app) + # FIXME : i18n, better info strings + logger.info(f"Deprovisionning {name} ...") + resource.deprovision(context=context) + + for name, infos in self.wanted.items(): + resource = AppResourceClassesByType[name](infos, self.app) + if name not in self.current.keys(): + # FIXME : i18n, better info strings + logger.info(f"Provisionning {name} ...") + else: + # FIXME : i18n, better info strings + logger.info(f"Updating {name} ...") + resource.provision_or_update(context=context) - - - - - def validate_resource_availability(self): - - for name, resource in self.resources.items(): - resource.validate_availability(context={}) - - def provision_or_update_resources(self): - - for name, resource in self.resources.items(): - logger.info("Running provision_or_upgrade for {self.type}") - resource.provision_or_update(context={}) - - -class AppResource(object): +class AppResource: def __init__(self, properties: Dict[str, Any], app: str): self.app = app for key, value in self.default_properties.items(): + if isinstance(value, str): + value = value.replace("__APP__", self.app) setattr(self, key, value) for key, value in properties.items(): + if isinstance(value, str): + value = value.replace("__APP__", self.app) setattr(self, key, value) def get_setting(self, key): @@ -78,7 +97,7 @@ class AppResource(object): from yunohost.app import app_setting app_setting(self.app, key, value=value) - def delete_setting(self, key, value): + def delete_setting(self, key): from yunohost.app import app_setting app_setting(self.app, key, delete=True) @@ -104,11 +123,12 @@ ynh_abort_if_errors write_to_file(script_path, script) - print(env_) + #print(env_) # FIXME : use the hook_exec_with_debug_instructions_stuff ret, _ = hook_exec(script_path, env=env_) - print(ret) + + #print(ret) class WebpathResource(AppResource): @@ -137,20 +157,28 @@ class WebpathResource(AppResource): def validate_availability(self, context): from yunohost.app import _assert_no_conflicting_apps - domain = self.get_setting("domain") path = self.get_setting("path") if not self.full_domain else "/" _assert_no_conflicting_apps(domain, path, ignore_app=self.app) def provision_or_update(self, context: Dict): - # Nothing to do ? Just setting the domain/path during install - # already provisions it ... - return # FIXME + from yunohost.permission import ( + permission_url, + user_permission_update, + permission_sync_to_user, + ) + + if context.get("action") == "install": + permission_url(f"{self.app}.main", url="/", sync_perm=False) + user_permission_update(f"{self.app}.main", show_tile=True, sync_perm=False) + permission_sync_to_user() def deprovision(self, context: Dict): self.delete_setting("domain") self.delete_setting("path") + # FIXME : theoretically here, should also remove the url in the main permission ? + # but is that worth the trouble ? class SystemuserAppResource(AppResource): @@ -173,7 +201,7 @@ class SystemuserAppResource(AppResource): priority = 20 default_properties = { - "allow_ssh": [] + "allow_ssh": [], "allow_sftp": [] } @@ -190,37 +218,37 @@ class SystemuserAppResource(AppResource): def provision_or_update(self, context: Dict): - if os.system(f"getent passwd {self.app} &>/dev/null") != 0: + if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): + # FIXME: improve error handling ? cmd = f"useradd --system --user-group {self.app}" os.system(cmd) - if os.system(f"getent passwd {self.app} &>/dev/null") == 0: - raise YunohostError(f"Failed to create system user for {self.app}") + if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): + raise YunohostError(f"Failed to create system user for {self.app}", raw_msg=True) + + groups = set(check_output(f"groups {self.app}").strip().split()[2:]) - groups = [] if self.allow_ssh: - groups.append("ssh.app") + groups.add("ssh.app") if self.allow_sftp: - groups.append("sftp.app") - groups = - - cmd = f"usermod -a -G {groups} {self.app}" - # FIXME : handle case where group gets removed - os.system(cmd) - -# useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account" -# for group in $groups; do -# usermod -a -G "$group" "$username" -# done - - -# | arg: -g, --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp) + groups.add("sftp.app") + os.system(f"usermod -G {','.join(groups)} {self.app}") def deprovision(self, context: Dict): - self._run_script("deprovision", - f'ynh_system_user_delete "{self.username}"') + if check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): + os.system(f"deluser {self.app} >/dev/null") + if check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): + raise YunohostError(f"Failed to delete system user for {self.app}") + + if check_output(f"getent group {self.app} &>/dev/null || true").strip(): + os.system(f"delgroup {self.app} >/dev/null") + if check_output(f"getent group {self.app} &>/dev/null || true").strip(): + raise YunohostError(f"Failed to delete system user for {self.app}") + + # FIXME : better logging and error handling, add stdout/stderr from the deluser/delgroup commands... + # # Check if the user exists on the system #if os.system(f"getent passwd {self.username} &>/dev/null") != 0: @@ -288,7 +316,7 @@ class InstalldirAppResource(AppResource): group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0) perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0" - chmod(self.dir, oct(int(perm_octal))) + chmod(self.dir, int(perm_octal)) chown(self.dir, owner, group) self.set_setting("install_dir", self.dir) @@ -348,7 +376,7 @@ class DatadirAppResource(AppResource): group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0) perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0" - chmod(self.dir, oct(int(perm_octal))) + chmod(self.dir, int(perm_octal)) chown(self.dir, owner, group) self.set_setting("data_dir", self.dir) From dfc37a48c382c7adf6cb2e1dcecf63d2baa580d6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 2 Jan 2022 03:30:57 +0100 Subject: [PATCH 15/54] manifestv2: forget about webpath ressource, replace with permissions ressource --- src/app.py | 69 +++++++++----------- src/utils/resources.py | 142 ++++++++++++++++++++++++++--------------- 2 files changed, 119 insertions(+), 92 deletions(-) diff --git a/src/app.py b/src/app.py index d2aa0fa65..5e67f620b 100644 --- a/src/app.py +++ b/src/app.py @@ -859,13 +859,13 @@ def app_install( # If packaging_format v2+, save all install questions as settings if packaging_format >= 2: - for arg_name, arg_value in args.items(): + for question in questions: - # ... except is_public .... - if arg_name == "is_public": + # Except user-provider passwords + if question.type == "password": continue - app_settings[arg_name] = arg_value + app_settings[question.name] = question.value _set_app_settings(app_instance_name, app_settings) @@ -878,36 +878,27 @@ def app_install( recursive=True, ) - # Initialize the main permission for the app - # The permission is initialized with no url associated, and with tile disabled - # For web app, the root path of the app will be added as url and the tile - # will be enabled during the app install. C.f. 'app_register_url()' below - # or the webpath resource - if packaging_format >= 2: - if args.get("init_permission_main"): - init_main_perm_allowed = args.get("init_permission_main") - else: - init_main_perm_allowed = ["visitors"] if not args.get("is_public") else ["all_users"] - - else: - init_main_perm_allowed = ["all_users"] - - permission_create( - app_instance_name + ".main", - allowed=init_main_perm_allowed, - label=label if label else manifest["name"], - show_tile=False, - protected=False, - ) - if packaging_format >= 2: from yunohost.utils.resources import AppResourceManager try: - AppResourceManager(app_instance_name, wanted=manifest["resources"], current={}).apply() + AppResourceManager(app_instance_name, wanted=manifest, current={}).apply() except Exception: # FIXME : improve error handling .... - AppResourceManager(app_instance_name, wanted={}, current=manifest["resources"]).apply() + AppResourceManager(app_instance_name, wanted={}, current=manifest).apply() raise + else: + # Initialize the main permission for the app + # The permission is initialized with no url associated, and with tile disabled + # For web app, the root path of the app will be added as url and the tile + # will be enabled during the app install. C.f. 'app_register_url()' below + # or the webpath resource + permission_create( + app_instance_name + ".main", + allowed=["all_users"], + label=label if label else manifest["name"], + show_tile=False, + protected=False, + ) # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( @@ -1008,15 +999,15 @@ def app_install( if packaging_format >= 2: from yunohost.utils.resources import AppResourceManager try: - AppResourceManager(app_instance_name, wanted={}, current=manifest["resources"]).apply() + AppResourceManager(app_instance_name, wanted={}, current=manifest).apply() except Exception: # FIXME : improve error handling .... raise - - # Remove all permission in LDAP - for permission_name in user_permission_list()["permissions"].keys(): - if permission_name.startswith(app_instance_name + "."): - permission_delete(permission_name, force=True, sync_perm=False) + else: + # Remove all permission in LDAP + for permission_name in user_permission_list()["permissions"].keys(): + if permission_name.startswith(app_instance_name + "."): + permission_delete(permission_name, force=True, sync_perm=False) if remove_retcode != 0: msg = m18n.n("app_not_properly_removed", app=app_instance_name) @@ -1116,18 +1107,18 @@ def app_remove(operation_logger, app, purge=False): finally: shutil.rmtree(tmp_workdir_for_app) - # Remove all permission in LDAP - for permission_name in user_permission_list(apps=[app])["permissions"].keys(): - permission_delete(permission_name, force=True, sync_perm=False) - packaging_format = manifest["packaging_format"] if packaging_format >= 2: try: from yunohost.utils.resources import AppResourceManager - AppResourceManager(app, wanted={}, current=manifest["resources"]).apply() + AppResourceManager(app, wanted={}, current=manifest).apply() except Exception: # FIXME : improve error handling .... raise + else: + # Remove all permission in LDAP + for permission_name in user_permission_list(apps=[app])["permissions"].keys(): + permission_delete(permission_name, force=True, sync_perm=False) if os.path.exists(app_setting_path): shutil.rmtree(app_setting_path) diff --git a/src/utils/resources.py b/src/utils/resources.py index b0956a2d0..23c1a7a5a 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -41,29 +41,23 @@ class AppResourceManager: def __init__(self, app: str, current: Dict, wanted: Dict): self.app = app - self.current = current - self.wanted = wanted + self.current = current.get("resources", {}) + self.wanted = wanted.get("resources", {}) + + # c.f. the permission ressources where we need the app label >_> + self.wanted_manifest = wanted def apply(self, **context): - for name, infos in self.wanted.items(): - resource = AppResourceClassesByType[name](infos, self.app) - # FIXME: not a great place to check this because here - # we already started an operation - # We should find a way to validate this before actually starting - # the install procedure / theoperation log - if name not in self.current.keys(): - resource.validate_availability(context=context) - for name, infos in reversed(self.current.items()): if name not in self.wanted.keys(): - resource = AppResourceClassesByType[name](infos, self.app) + resource = AppResourceClassesByType[name](infos, self.app, self) # FIXME : i18n, better info strings logger.info(f"Deprovisionning {name} ...") resource.deprovision(context=context) for name, infos in self.wanted.items(): - resource = AppResourceClassesByType[name](infos, self.app) + resource = AppResourceClassesByType[name](infos, self.app, self) if name not in self.current.keys(): # FIXME : i18n, better info strings logger.info(f"Provisionning {name} ...") @@ -75,9 +69,10 @@ class AppResourceManager: class AppResource: - def __init__(self, properties: Dict[str, Any], app: str): + def __init__(self, properties: Dict[str, Any], app: str, manager: str): self.app = app + self.manager = manager for key, value in self.default_properties.items(): if isinstance(value, str): @@ -101,9 +96,6 @@ class AppResource: from yunohost.app import app_setting app_setting(self.app, key, delete=True) - def validate_availability(self, context: Dict): - pass - def _run_script(self, action, script, env={}, user="root"): from yunohost.app import _make_tmp_workdir_for_app, _make_environment_for_app_script @@ -131,7 +123,7 @@ ynh_abort_if_errors #print(ret) -class WebpathResource(AppResource): +class PermissionsResource(AppResource): """ is_provisioned -> main perm exists is_available -> perm urls do not conflict @@ -147,38 +139,98 @@ class WebpathResource(AppResource): restore -> handled by the core, should be integrated in there (restore .ldif/yml?) """ - type = "webpath" + type = "permissions" priority = 10 default_properties = { - "full_domain": False, } - def validate_availability(self, context): + default_perm_properties = { + "url": None, + "additional_urls": [], + "auth_header": True, + "allowed": None, + "show_tile": None, # To be automagically set to True by default if an url is defined and show_tile not provided + "protected": False, + } - from yunohost.app import _assert_no_conflicting_apps - domain = self.get_setting("domain") - path = self.get_setting("path") if not self.full_domain else "/" - _assert_no_conflicting_apps(domain, path, ignore_app=self.app) + def __init__(self, properties: Dict[str, Any], *args, **kwargs): + + for perm, infos in properties.items(): + properties[perm] = copy.copy(self.default_perm_properties) + properties[perm].update(infos) + if properties[perm]["show_tile"] is None: + properties[perm]["show_tile"] = bool(properties[perm]["url"]) + + if isinstance(properties["main"]["url"], str) and properties["main"]["url"] != "/": + raise YunohostError("URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app") + + super().__init__({"permissions": properties}, *args, **kwargs) def provision_or_update(self, context: Dict): from yunohost.permission import ( - permission_url, + permission_create, + #permission_url, + permission_delete, + user_permission_list, user_permission_update, permission_sync_to_user, ) - if context.get("action") == "install": - permission_url(f"{self.app}.main", url="/", sync_perm=False) - user_permission_update(f"{self.app}.main", show_tile=True, sync_perm=False) - permission_sync_to_user() + # Delete legacy is_public setting if not already done + self.delete_setting(f"is_public") + + existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"] + for perm in existing_perms: + if perm.split(".") not in self.permissions.keys(): + permission_delete(perm, force=True, sync_perm=False) + + for perm, infos in self.permissions.items(): + if f"{self.app}.{perm}" not in existing_perms: + # Use the 'allowed' key from the manifest, + # or use the 'init_{perm}_permission' from the install questions + # which is temporarily saved as a setting as an ugly hack to pass the info to this piece of code... + init_allowed = infos["allowed"] or self.get_setting(f"init_{perm}_permission") or [] + permission_create( + f"{self.app}.{perm}", + allowed=init_allowed, + # This is why the ugly hack with self.manager and wanted_manifest exists >_> + label=self.manager.wanted_manifest["name"] if perm == "main" else perm, + url=infos["url"], + additional_urls=infos["additional_urls"], + auth_header=infos["auth_header"], + sync_perm=False, + ) + self.delete_setting(f"init_{perm}_permission") + + user_permission_update( + f"{self.app}.{perm}", + show_tile=infos["show_tile"], + protected=infos["protected"], + sync_perm=False + ) + else: + pass + # FIXME : current implementation of permission_url is hell for + # easy declarativeness of additional_urls >_> ... + #permission_url(f"{self.app}.{perm}", url=infos["url"], auth_header=infos["auth_header"], sync_perm=False) + + permission_sync_to_user() def deprovision(self, context: Dict): - self.delete_setting("domain") - self.delete_setting("path") - # FIXME : theoretically here, should also remove the url in the main permission ? - # but is that worth the trouble ? + + from yunohost.permission import ( + permission_delete, + user_permission_list, + permission_sync_to_user, + ) + + existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"] + for perm in existing_perms: + permission_delete(perm, force=True, sync_perm=False) + + permission_sync_to_user() class SystemuserAppResource(AppResource): @@ -205,19 +257,11 @@ class SystemuserAppResource(AppResource): "allow_sftp": [] } - def validate_availability(self, context): - pass - # FIXME : do we care if user already exists ? shouldnt we assume that user $app corresponds to the app ...? - - # FIXME : but maybe we should at least check that no corresponding yunohost user exists - - #if os.system(f"getent passwd {self.username} &>/dev/null") != 0: - # raise YunohostValidationError(f"User {self.username} already exists") - #if os.system(f"getent group {self.username} &>/dev/null") != 0: - # raise YunohostValidationError(f"Group {self.username} already exists") - def provision_or_update(self, context: Dict): + # FIXME : validate that no yunohost user exists with that name? + # and/or that no system user exists during install ? + if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): # FIXME: improve error handling ? cmd = f"useradd --system --user-group {self.app}" @@ -291,9 +335,6 @@ class InstalldirAppResource(AppResource): # FIXME: change default dir to /opt/stuff if app ain't a webapp ... # FIXME: what do in a scenario where the location changed - def validate_availability(self, context): - pass - def provision_or_update(self, context: Dict): current_install_dir = self.get_setting("install_dir") @@ -354,11 +395,6 @@ class DatadirAppResource(AppResource): "group": "__APP__:rx", } - def validate_availability(self, context): - pass - # Nothing to do ? If datadir already exists then it may be legit data - # from a previous install - def provision_or_update(self, context: Dict): current_data_dir = self.get_setting("data_dir") From 4c6786e8afe13ad703d80c6ddaf39630b3aacd70 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 3 Jan 2022 18:02:41 +0100 Subject: [PATCH 16/54] manifestv2, appresources: Implement a more generic 'apply' mecanism with possible rollback --- src/app.py | 34 ++++++---- src/backup.py | 10 +++ src/tests/test_app_resources.py | 115 ++++++++++++++++++++++++++++++++ src/utils/resources.py | 97 ++++++++++++++++++++------- 4 files changed, 218 insertions(+), 38 deletions(-) create mode 100644 src/tests/test_app_resources.py diff --git a/src/app.py b/src/app.py index 2e7c428d7..2f3f3e835 100644 --- a/src/app.py +++ b/src/app.py @@ -546,6 +546,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if manifest["packaging_format"] >= 2: if no_safety_backup: + # FIXME: i18n logger.warning("Skipping the creation of a backup prior to the upgrade.") else: # FIXME: i18n @@ -570,8 +571,17 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False _assert_system_is_sane_for_app(manifest, "pre") + # We'll check that the app didn't brutally edit some system configuration + manually_modified_files_before_install = manually_modified_files() + app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) + # Attempt to patch legacy helpers ... + _patch_legacy_helpers(extracted_app_folder) + + # Apply dirty patch to make php5 apps compatible with php7 + _patch_legacy_php_versions(extracted_app_folder) + # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( app_instance_name, workdir=extracted_app_folder, action="upgrade" @@ -582,20 +592,19 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if manifest["packaging_format"] < 2: env_dict["NO_BACKUP_UPGRADE"] = "1" if no_safety_backup else "0" - # We'll check that the app didn't brutally edit some system configuration - manually_modified_files_before_install = manually_modified_files() - - # Attempt to patch legacy helpers ... - _patch_legacy_helpers(extracted_app_folder) - - # Apply dirty patch to make php5 apps compatible with php7 - _patch_legacy_php_versions(extracted_app_folder) - # Start register change on system related_to = [("app", app_instance_name)] operation_logger = OperationLogger("app_upgrade", related_to, env=env_dict) operation_logger.start() + if manifest["packaging_format"] >= 2: + from yunohost.utils.resources import AppResourceManager + try: + AppResourceManager(app_instance_name, wanted=manifest, current=app_dict["manifest"]).apply(rollback_if_failure=True) + except Exception: + # FIXME : improve error handling .... + raise + # Execute the app upgrade script upgrade_failed = True try: @@ -880,10 +889,9 @@ def app_install( if packaging_format >= 2: from yunohost.utils.resources import AppResourceManager try: - AppResourceManager(app_instance_name, wanted=manifest, current={}).apply() + AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(rollback_if_failure=True) except Exception: # FIXME : improve error handling .... - AppResourceManager(app_instance_name, wanted={}, current=manifest).apply() raise else: # Initialize the main permission for the app @@ -997,7 +1005,7 @@ def app_install( if packaging_format >= 2: from yunohost.utils.resources import AppResourceManager try: - AppResourceManager(app_instance_name, wanted={}, current=manifest).apply() + AppResourceManager(app_instance_name, wanted={}, current=manifest).apply(rollback_if_failure=False) except Exception: # FIXME : improve error handling .... raise @@ -1109,7 +1117,7 @@ def app_remove(operation_logger, app, purge=False): if packaging_format >= 2: try: from yunohost.utils.resources import AppResourceManager - AppResourceManager(app, wanted={}, current=manifest).apply() + AppResourceManager(app, wanted={}, current=manifest).apply(rollback_if_failure=False) except Exception: # FIXME : improve error handling .... raise diff --git a/src/backup.py b/src/backup.py index 1e16435b4..c2d921548 100644 --- a/src/backup.py +++ b/src/backup.py @@ -49,6 +49,7 @@ from yunohost.app import ( _is_installed, _make_environment_for_app_script, _make_tmp_workdir_for_app, + _get_manifest_of_app, ) from yunohost.hook import ( hook_list, @@ -1512,6 +1513,15 @@ class RestoreManager: operation_logger.extra["env"] = env_dict operation_logger.flush() + manifest = _get_manifest_of_app(app_dir_in_archive) + if manifest["packaging_format"] >= 2: + from yunohost.utils.resources import AppResourceManager + try: + AppResourceManager(app_instance_name, wanted=manifest, current={}).apply(rollback_if_failure=True) + except Exception: + # FIXME : improve error handling .... + raise + # Execute the app install script restore_failed = True try: diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py new file mode 100644 index 000000000..d3abddc7f --- /dev/null +++ b/src/tests/test_app_resources.py @@ -0,0 +1,115 @@ +import os +import pytest + +from yunohost.utils.resources import AppResource, AppResourceManager, AppResourceClassesByType + +dummyfile = "/tmp/dummyappresource-testapp" + + +class DummyAppResource(AppResource): + + type = "dummy" + + default_properties = { + "file": "/tmp/dummyappresource-__APP__", + "content": "foo", + } + + def provision_or_update(self, context): + + open(self.file, "w").write(self.content) + + if self.content == "forbiddenvalue": + raise Exception("Emeged you used the forbidden value!1!£&") + + def deprovision(self, context): + + os.system(f"rm -f {self.file}") + + +AppResourceClassesByType["dummy"] = DummyAppResource + + +def setup_function(function): + + clean() + + +def teardown_function(function): + + clean() + + +def clean(): + + os.system(f"rm -f {dummyfile}") + + +def test_provision_dummy(): + + current = {"resources": {}} + wanted = {"resources": {"dummy": {}}} + + assert not os.path.exists(dummyfile) + AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False) + assert open(dummyfile).read().strip() == "foo" + + +def test_deprovision_dummy(): + + current = {"resources": {"dummy": {}}} + wanted = {"resources": {}} + + open(dummyfile, "w").write("foo") + + assert open(dummyfile).read().strip() == "foo" + AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False) + assert not os.path.exists(dummyfile) + + +def test_provision_dummy_nondefaultvalue(): + + current = {"resources": {}} + wanted = {"resources": {"dummy": {"content": "bar"}}} + + assert not os.path.exists(dummyfile) + AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False) + assert open(dummyfile).read().strip() == "bar" + + +def test_update_dummy(): + + current = {"resources": {"dummy": {}}} + wanted = {"resources": {"dummy": {"content": "bar"}}} + + open(dummyfile, "w").write("foo") + + assert open(dummyfile).read().strip() == "foo" + AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False) + assert open(dummyfile).read().strip() == "bar" + + +def test_update_dummy_fail(): + + current = {"resources": {"dummy": {}}} + wanted = {"resources": {"dummy": {"content": "forbiddenvalue"}}} + + open(dummyfile, "w").write("foo") + + assert open(dummyfile).read().strip() == "foo" + with pytest.raises(Exception): + AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=False) + assert open(dummyfile).read().strip() == "forbiddenvalue" + + +def test_update_dummy_failwithrollback(): + + current = {"resources": {"dummy": {}}} + wanted = {"resources": {"dummy": {"content": "forbiddenvalue"}}} + + open(dummyfile, "w").write("foo") + + assert open(dummyfile).read().strip() == "foo" + with pytest.raises(Exception): + AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=True) + assert open(dummyfile).read().strip() == "foo" diff --git a/src/utils/resources.py b/src/utils/resources.py index 23c1a7a5a..4e9876655 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -41,30 +41,77 @@ class AppResourceManager: def __init__(self, app: str, current: Dict, wanted: Dict): self.app = app - self.current = current.get("resources", {}) - self.wanted = wanted.get("resources", {}) + self.current = current + self.wanted = wanted - # c.f. the permission ressources where we need the app label >_> - self.wanted_manifest = wanted + def apply(self, rollback_if_failure, **context): - def apply(self, **context): + todos = list(self.compute_todos()) + completed = [] + rollback = False + exception = None - for name, infos in reversed(self.current.items()): - if name not in self.wanted.keys(): - resource = AppResourceClassesByType[name](infos, self.app, self) - # FIXME : i18n, better info strings - logger.info(f"Deprovisionning {name} ...") - resource.deprovision(context=context) - - for name, infos in self.wanted.items(): - resource = AppResourceClassesByType[name](infos, self.app, self) - if name not in self.current.keys(): - # FIXME : i18n, better info strings - logger.info(f"Provisionning {name} ...") + for todo, name, old, new in todos: + try: + if todo == "deprovision": + # FIXME : i18n, better info strings + logger.info(f"Deprovisionning {name} ...") + old.deprovision(context=context) + elif todo == "provision": + logger.info(f"Provisionning {name} ...") + new.provision_or_update(context=context) + elif todo == "update": + logger.info(f"Updating {name} ...") + new.provision_or_update(context=context) + except Exception as e: + exception = e + # FIXME: better error handling ? display stacktrace ? + logger.warning(f"Failed to {todo} for {name} : {e}") + if rollback_if_failure: + rollback = True + completed.append((todo, name, old, new)) + break + else: + pass else: - # FIXME : i18n, better info strings - logger.info(f"Updating {name} ...") - resource.provision_or_update(context=context) + completed.append((todo, name, old, new)) + + if rollback: + for todo, name, old, new in completed: + try: + # (NB. here we want to undo the todo) + if todo == "deprovision": + # FIXME : i18n, better info strings + logger.info(f"Reprovisionning {name} ...") + old.provision_or_update(context=context) + elif todo == "provision": + logger.info(f"Deprovisionning {name} ...") + new.deprovision(context=context) + elif todo == "update": + logger.info(f"Reverting {name} ...") + old.provision_or_update(context=context) + except Exception as e: + # FIXME: better error handling ? display stacktrace ? + logger.error(f"Failed to rollback {name} : {e}") + + if exception: + raise exception + + def compute_todos(self): + + for name, infos in reversed(self.current["resources"].items()): + if name not in self.wanted["resources"].keys(): + resource = AppResourceClassesByType[name](infos, self.app, self) + yield ("deprovision", name, resource, None) + + for name, infos in self.wanted["resources"].items(): + wanted_resource = AppResourceClassesByType[name](infos, self.app, self) + if name not in self.current["resources"].keys(): + yield ("provision", name, None, wanted_resource) + else: + infos_ = self.current["resources"][name] + current_resource = AppResourceClassesByType[name](infos_, self.app, self) + yield ("update", name, current_resource, wanted_resource) class AppResource: @@ -179,7 +226,7 @@ class PermissionsResource(AppResource): ) # Delete legacy is_public setting if not already done - self.delete_setting(f"is_public") + self.delete_setting("is_public") existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"] for perm in existing_perms: @@ -195,8 +242,8 @@ class PermissionsResource(AppResource): permission_create( f"{self.app}.{perm}", allowed=init_allowed, - # This is why the ugly hack with self.manager and wanted_manifest exists >_> - label=self.manager.wanted_manifest["name"] if perm == "main" else perm, + # This is why the ugly hack with self.manager exists >_> + label=self.manager.wanted["name"] if perm == "main" else perm, url=infos["url"], additional_urls=infos["additional_urls"], auth_header=infos["auth_header"], @@ -499,7 +546,7 @@ class AptDependenciesAppResource(AppResource): "ynh_remove_app_dependencies") -class PortAppResource(AppResource): +class PortResource(AppResource): """ is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting) is_available -> true @@ -520,7 +567,7 @@ class PortAppResource(AppResource): default_properties = { "default": 1000, - "type": "internal", # FIXME : implement logic for exposed port (allow/disallow in firewall ?) + "expose": False, # FIXME : implement logic for exposed port (allow/disallow in firewall ?) } def _port_is_used(self, port): From 55963bef2309dd37566d8e57a9ea0ec9e5518bfa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jan 2022 21:06:05 +0100 Subject: [PATCH 17/54] Cleanup --- src/utils/resources.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 4e9876655..b5ddb148e 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -300,8 +300,8 @@ class SystemuserAppResource(AppResource): priority = 20 default_properties = { - "allow_ssh": [], - "allow_sftp": [] + "allow_ssh": False, + "allow_sftp": False } def provision_or_update(self, context: Dict): @@ -494,11 +494,6 @@ class DatadirAppResource(AppResource): # "main": {"url": "?", "sha256sum": "?", "predownload": True} # } # -# def validate_availability(self, context): -# # ? FIXME -# # call request.head on the url idk -# pass -# # def provision_or_update(self, context: Dict): # # FIXME # return @@ -528,11 +523,6 @@ class AptDependenciesAppResource(AppResource): "extras": {} } - def validate_availability(self, context): - # ? FIXME - # call helpers idk ... - pass - def provision_or_update(self, context: Dict): # FIXME : implement 'extras' management @@ -568,6 +558,7 @@ class PortResource(AppResource): default_properties = { "default": 1000, "expose": False, # FIXME : implement logic for exposed port (allow/disallow in firewall ?) + # "protocol": "tcp", } def _port_is_used(self, port): @@ -617,11 +608,6 @@ class PortResource(AppResource): # "type": "mysql" # } # -# def validate_availability(self, context): -# # FIXME : checking availability sort of imply that mysql / postgresql is installed -# # or we gotta make sure mariadb-server or postgresql is gonna be installed (apt resource) -# pass -# # def provision_or_update(self, context: str): # raise NotImplementedError() # From 33d4dc9749a567c3526599109f020e81d6686cad Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jan 2022 21:41:02 +0100 Subject: [PATCH 18/54] apt resources: Yoloimplement extras repo management --- src/utils/resources.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index b5ddb148e..c28f09390 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -30,7 +30,7 @@ from moulinette.utils.filesystem import ( rm, ) -from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.utils.error import YunohostError from yunohost.hook import hook_exec logger = getActionLogger("yunohost.app_resources") @@ -210,7 +210,7 @@ class PermissionsResource(AppResource): properties[perm]["show_tile"] = bool(properties[perm]["url"]) if isinstance(properties["main"]["url"], str) and properties["main"]["url"] != "/": - raise YunohostError("URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app") + raise YunohostValidationError("URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app") super().__init__({"permissions": properties}, *args, **kwargs) @@ -523,17 +523,25 @@ class AptDependenciesAppResource(AppResource): "extras": {} } + def __init__(self, properties: Dict[str, Any], *args, **kwargs): + + for key, values in properties.get("extras", {}).items(): + if not all(isinstance(values.get(k), str) for k in ["repo", "key", "packages"]): + raise YunohostError("In apt resource in the manifest: 'extras' repo should have the keys 'repo', 'key' and 'packages' defined and be strings") + + super().__init__(properties, *args, **kwargs) + def provision_or_update(self, context: Dict): - # FIXME : implement 'extras' management - self._run_script("provision_or_update", - "ynh_install_app_dependencies $apt_dependencies", - {"apt_dependencies": self.packages}) + script = [f"ynh_install_app_dependencies {self.packages}"] + for repo, values in self.extras.items(): + script += [f"ynh_install_extra_app_dependencies --repo='{values['repo']}' --key='{values['key']}' --package='{values['packages']}'"] + + self._run_script("provision_or_update", '\n'.join(script)) def deprovision(self, context: Dict): - self._run_script("deprovision", - "ynh_remove_app_dependencies") + self._run_script("deprovision", "ynh_remove_app_dependencies") class PortResource(AppResource): From 13fb471f29b57a024df04f8bfbd20e299c4c066d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jan 2022 21:58:11 +0100 Subject: [PATCH 19/54] install_dir resource: automigrate from final path value --- src/utils/resources.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index c28f09390..73dd98380 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -374,17 +374,15 @@ class InstalldirAppResource(AppResource): default_properties = { "dir": "/var/www/__APP__", # FIXME or choose to move this elsewhere nowadays idk... - "alias": None, "owner": "__APP__:rx", "group": "__APP__:rx", } # FIXME: change default dir to /opt/stuff if app ain't a webapp ... - # FIXME: what do in a scenario where the location changed def provision_or_update(self, context: Dict): - current_install_dir = self.get_setting("install_dir") + current_install_dir = self.get_setting("install_dir") or self.get_setting("final_path") # If during install, /var/www/$app already exists, assume that it's okay to remove and recreate it # FIXME : is this the right thing to do ? @@ -393,6 +391,7 @@ class InstalldirAppResource(AppResource): if not os.path.isdir(self.dir): # Handle case where install location changed, in which case we shall move the existing install dir + # FIXME: confirm that's what we wanna do if current_install_dir and os.path.isdir(current_install_dir): shutil.move(current_install_dir, self.dir) else: @@ -406,10 +405,10 @@ class InstalldirAppResource(AppResource): chmod(self.dir, int(perm_octal)) chown(self.dir, owner, group) + # FIXME: shall we apply permissions recursively ? self.set_setting("install_dir", self.dir) - if self.alias: - self.set_setting(self.alias, self.dir) + self.delete_setting("final_path") # Legacy def deprovision(self, context: Dict): # FIXME : check that self.dir has a sensible value to prevent catastrophes From b68e99b2c0e4d43f2cfad184c3bc9ef38ccdd37a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jan 2022 23:02:50 +0100 Subject: [PATCH 20/54] app resources: Yoloimplement support for db resources --- src/utils/resources.py | 107 ++++++++++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 28 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 73dd98380..45c76acd8 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -593,33 +593,84 @@ class PortResource(AppResource): self.delete_setting("port") -#class DBAppResource(AppResource): -# """ -# is_provisioned -> setting db_user, db_name, db_pwd exists -# is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it) -# -# provision -> setup the db + init the setting -# update -> ?? -# -# deprovision -> delete the db -# -# deep_clean -> ... idk look into any db name that would not be related to any app ... -# -# backup -> dump db -# restore -> setup + inject db dump -# """ -# -# type = "db" -# -# default_properties = { -# "type": "mysql" -# } -# -# def provision_or_update(self, context: str): -# raise NotImplementedError() -# -# def deprovision(self, context: Dict): -# raise NotImplementedError() -# +class DBAppResource(AppResource): + """ + is_provisioned -> setting db_user, db_name, db_pwd exists + is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it) + + provision -> setup the db + init the setting + update -> ?? + + deprovision -> delete the db + + deep_clean -> ... idk look into any db name that would not be related to any app ... + + backup -> dump db + restore -> setup + inject db dump + """ + + type = "db" + + default_properties = { + "type": None, + } + + def __init__(self, properties: Dict[str, Any], *args, **kwargs): + + if "type" not in properties or properties["type"] not in ["mysql", "postgresql"]: + raise YunohostError("Specifying the type of db ('mysql' or 'postgresql') is mandatory for db resources") + + super().__init__(properties, *args, **kwargs) + + def db_exists(self, db_name): + + if self.type == "mysql": + return os.system(f"mysqlshow '{db_name}' >/dev/null 2>/dev/null") == 0 + elif self.type == "postgresql": + return os.system(f"sudo --login --user=postgres psql -c '' '{db_name}' >/dev/null 2>/dev/null") == 0 + else: + return False + + def provision_or_update(self, context: str): + + # This is equivalent to ynh_sanitize_dbid + db_name = self.app.replace('-', '_').replace('.', '_') + db_user = db_name + self.set_setting("db_name", db_name) + self.set_setting("db_user", db_user) + + db_pwd = None + if self.get_setting("db_pwd"): + db_pwd = self.get_setting("db_pwd") + else: + # Legacy setting migration + legacypasswordsetting = "psqlpwd" if self.type == "postgresql" else "mysqlpwd" + if self.get_setting(legacypasswordsetting): + db_pwd = self.get_setting(legacypasswordsetting) + self.delete_setting(legacypasswordsetting) + self.set_setting("db_pwd", db_pwd) + + if not db_pwd: + from moulinette.utils.text import random_ascii + db_pwd = random_ascii(24) + self.set_setting("db_pwd", db_pwd) + + if not self.db_exists(db_name): + + if self.type == "mysql": + self._run_script("provision", f"ynh_mysql_create_db '{db_name}' '{db_user}' '{db_pwd}'") + elif self.type == "postgresql": + self._run_script("provision", f"ynh_psql_create_user '{db_user}' '{db_pwd}'; ynh_psql_create_db '{db_name}' '{db_user}'") + + def deprovision(self, context: Dict): + + db_name = self.app.replace('-', '_').replace('.', '_') + db_user = db_name + + if self.type == "mysql": + self._run_script("deprovision", f"ynh_mysql_remove_db '{db_name}' '{db_user}'") + elif self.type == "postgresql": + self._run_script("deprovision", f"ynh_psql_remove_db '{db_name}' '{db_user}'") + AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()} From 8bc23e6f595f3563eac0389c0efaada1e8d15296 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 15 Jan 2022 19:18:21 +0100 Subject: [PATCH 21/54] app resources: Yoloimplement some tests for each resources --- src/tests/test_app_resources.py | 197 ++++++++++++++++++++++++++++++++ src/utils/resources.py | 16 ++- 2 files changed, 208 insertions(+), 5 deletions(-) diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index d3abddc7f..69e93a5ed 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -1,6 +1,9 @@ import os import pytest +from moulinette.utils.process import check_output + +from yunohost.app import app_setting from yunohost.utils.resources import AppResource, AppResourceManager, AppResourceClassesByType dummyfile = "/tmp/dummyappresource-testapp" @@ -43,6 +46,7 @@ def teardown_function(function): def clean(): os.system(f"rm -f {dummyfile}") + os.system("apt remove lolcat sl nyancat yarn") def test_provision_dummy(): @@ -113,3 +117,196 @@ def test_update_dummy_failwithrollback(): with pytest.raises(Exception): AppResourceManager("testapp", current=current, wanted=wanted).apply(rollback_if_failure=True) assert open(dummyfile).read().strip() == "foo" + + +def test_resource_system_user(): + + r = AppResourceClassesByType["system_user"] + + conf = {} + + assert os.system("getent passwd testapp &>/dev/null") != 0 + + r(conf, "testapp").provision_or_update() + + assert os.system("getent passwd testapp &>/dev/null") == 0 + assert os.system("getent group testapp | grep -q sftp.app") != 0 + + conf["allow_sftp"] = True + r(conf, "testapp").provision_or_update() + + assert os.system("getent passwd testapp &>/dev/null") == 0 + assert os.system("getent group testapp | grep -q sftp.app") == 0 + + r(conf, "testapp").deprovision() + + assert os.system("getent passwd testapp &>/dev/null") != 0 + + +def test_resource_install_dir(): + + r = AppResourceClassesByType["install_dir"] + conf = {} + + # FIXME: should also check settings ? + # FIXME: should also check automigrate from final_path + # FIXME: should also test changing the install folder location ? + + assert not os.path.exists("/var/www/testapp") + + r(conf, "testapp").provision_or_update() + + assert os.path.exists("/var/www/testapp") + unixperms = check_output("ls -ld /var/www/testapp").split() + assert unixperms[0] == "dr-xr-x---" + assert unixperms[2] == "testapp" + assert unixperms[3] == "testapp" + + conf["group"] = "__APP__:rwx" + conf["group"] = "www-data:x" + + r(conf, "testapp").provision_or_update() + + assert os.path.exists("/var/www/testapp") + unixperms = check_output("ls -ld /var/www/testapp").split() + assert unixperms[0] == "drwx--x---" + assert unixperms[2] == "testapp" + assert unixperms[3] == "www-data" + + r(conf, "testapp").deprovision() + + assert not os.path.exists("/var/www/testapp") + + +def test_resource_data_dir(): + + r = AppResourceClassesByType["data_dir"] + conf = {} + + assert not os.path.exists("/home/yunohost.app/testapp") + + r(conf, "testapp").provision_or_update() + + assert os.path.exists("/home/yunohost.app/testapp") + unixperms = check_output("ls -ld /home/yunohost.app/testapp").split() + assert unixperms[0] == "dr-xr-x---" + assert unixperms[2] == "testapp" + assert unixperms[3] == "testapp" + + conf["group"] = "__APP__:rwx" + conf["group"] = "www-data:x" + + r(conf, "testapp").provision_or_update() + + assert os.path.exists("/home/yunohost.app/testapp") + unixperms = check_output("ls -ld /home/yunohost.app/testapp").split() + assert unixperms[0] == "drwx--x---" + assert unixperms[2] == "testapp" + assert unixperms[3] == "www-data" + + r(conf, "testapp").deprovision() + + assert not os.path.exists("/home/yunohost.app/testapp") + + +def test_resource_port(): + + r = AppResourceClassesByType["port"] + conf = {} + + assert not app_setting("testapp", "port") + + r(conf, "testapp").provision_or_update() + + assert app_setting("testapp", "port") + + r(conf, "testapp").deprovision() + + assert not app_setting("testapp", "port") + + +def test_resource_database(): + + r = AppResourceClassesByType["database"] + conf = {"type": "mysql"} + + assert os.system("mysqlshow 'testapp' >/dev/null 2>/dev/null") != 0 + assert not app_setting("testapp", "db_name") + assert not app_setting("testapp", "db_user") + assert not app_setting("testapp", "db_pwd") + + r(conf, "testapp").provision_or_update() + + assert os.system("mysqlshow 'testapp' >/dev/null 2>/dev/null") == 0 + assert app_setting("testapp", "db_name") + assert app_setting("testapp", "db_user") + assert app_setting("testapp", "db_pwd") + + r(conf, "testapp").deprovision() + + assert os.system("mysqlshow 'testapp' >/dev/null 2>/dev/null") != 0 + assert not app_setting("testapp", "db_name") + assert not app_setting("testapp", "db_user") + assert not app_setting("testapp", "db_pwd") + + +def test_resource_apt(): + + r = AppResourceClassesByType["apt"] + conf = { + "packages": "nyancat, sl", + "extras": { + "yarn": { + "repo": "deb https://dl.yarnpkg.com/debian/ stable main", + "key": "https://dl.yarnpkg.com/debian/pubkey.gpg", + "packages": "yarn", + } + } + } + + assert os.system("dpkg --list | grep -q 'ii *nyancat'") != 0 + assert os.system("dpkg --list | grep -q 'ii *sl'") != 0 + assert os.system("dpkg --list | grep -q 'ii *yarn'") != 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat'") != 0 + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") != 0 + + r(conf, "testapp").provision_or_update() + + assert os.system("dpkg --list | grep -q 'ii *nyancat'") == 0 + assert os.system("dpkg --list | grep -q 'ii *sl'") == 0 + assert os.system("dpkg --list | grep -q 'ii *yarn'") == 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat'") != 0 # Lolcat shouldnt be installed yet + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") == 0 + + conf["packages"] += ", lolcat" + r(conf, "testapp").provision_or_update() + + assert os.system("dpkg --list | grep -q 'ii *nyancat'") != 0 + assert os.system("dpkg --list | grep -q 'ii *sl'") != 0 + assert os.system("dpkg --list | grep -q 'ii *yarn'") != 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat'") == 0 + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") != 0 + + r(conf, "testapp").deprovision() + + assert os.system("dpkg --list | grep -q 'ii *nyancat'") != 0 + assert os.system("dpkg --list | grep -q 'ii *sl'") != 0 + assert os.system("dpkg --list | grep -q 'ii *yarn'") != 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat'") != 0 + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") != 0 + + +def test_resource_permissions(): + + r = AppResourceClassesByType["permissions"] + conf = { + "main": { + "url": "/", + "allowed": "visitors" + # protected? + } + } + + pass + + diff --git a/src/utils/resources.py b/src/utils/resources.py index 45c76acd8..b4ec95bf8 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -116,7 +116,7 @@ class AppResourceManager: class AppResource: - def __init__(self, properties: Dict[str, Any], app: str, manager: str): + def __init__(self, properties: Dict[str, Any], app: str, manager=None): self.app = app self.manager = manager @@ -203,6 +203,8 @@ class PermissionsResource(AppResource): def __init__(self, properties: Dict[str, Any], *args, **kwargs): + # FIXME : if url != None, we should check that there's indeed a domain/path defined ? ie that app is a webapp + for perm, infos in properties.items(): properties[perm] = copy.copy(self.default_perm_properties) properties[perm].update(infos) @@ -210,7 +212,7 @@ class PermissionsResource(AppResource): properties[perm]["show_tile"] = bool(properties[perm]["url"]) if isinstance(properties["main"]["url"], str) and properties["main"]["url"] != "/": - raise YunohostValidationError("URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app") + raise YunohostError("URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app") super().__init__({"permissions": properties}, *args, **kwargs) @@ -414,6 +416,7 @@ class InstalldirAppResource(AppResource): # FIXME : check that self.dir has a sensible value to prevent catastrophes if os.path.isdir(self.dir): rm(self.dir, recursive=True) + # FIXME : in fact we should delete settings to be consistent class DatadirAppResource(AppResource): @@ -468,6 +471,7 @@ class DatadirAppResource(AppResource): pass #if os.path.isdir(self.dir): # rm(self.dir, recursive=True) + # FIXME : in fact we should delete settings to be consistent # @@ -563,7 +567,7 @@ class PortResource(AppResource): priority = 70 default_properties = { - "default": 1000, + "default": 5000, "expose": False, # FIXME : implement logic for exposed port (allow/disallow in firewall ?) # "protocol": "tcp", } @@ -593,7 +597,7 @@ class PortResource(AppResource): self.delete_setting("port") -class DBAppResource(AppResource): +class DatabaseAppResource(AppResource): """ is_provisioned -> setting db_user, db_name, db_pwd exists is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it) @@ -609,7 +613,7 @@ class DBAppResource(AppResource): restore -> setup + inject db dump """ - type = "db" + type = "database" default_properties = { "type": None, @@ -672,5 +676,7 @@ class DBAppResource(AppResource): elif self.type == "postgresql": self._run_script("deprovision", f"ynh_psql_remove_db '{db_name}' '{db_user}'") + # FIXME : in fact we should delete settings to be consistent + AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()} From 011c2059a55b10a56ce78e2d5cb17bd3d128b47a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 16 Jan 2022 03:34:53 +0100 Subject: [PATCH 22/54] appresources: fixes /o\ --- src/tests/test_app_resources.py | 88 +++++++++++++++++++-------------- src/utils/resources.py | 43 ++++++++-------- 2 files changed, 73 insertions(+), 58 deletions(-) diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index 69e93a5ed..638f6b119 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -37,6 +37,11 @@ def setup_function(function): clean() + os.system("mkdir /etc/yunohost/apps/testapp") + os.system("echo 'id: testapp' > /etc/yunohost/apps/testapp/settings.yml") + os.system("echo 'packaging_format = 2' > /etc/yunohost/apps/testapp/manifest.toml") + os.system("echo 'id = \"testapp\"' >> /etc/yunohost/apps/testapp/manifest.toml") + def teardown_function(function): @@ -46,7 +51,11 @@ def teardown_function(function): def clean(): os.system(f"rm -f {dummyfile}") - os.system("apt remove lolcat sl nyancat yarn") + os.system("rm -rf /etc/yunohost/apps/testapp") + os.system("rm -rf /var/www/testapp") + os.system("rm -rf /home/yunohost.app/testapp") + os.system("apt remove lolcat sl nyancat yarn >/dev/null 2>/dev/null") + os.system("userdel testapp 2>/dev/null") def test_provision_dummy(): @@ -125,28 +134,28 @@ def test_resource_system_user(): conf = {} - assert os.system("getent passwd testapp &>/dev/null") != 0 + assert os.system("getent passwd testapp 2>/dev/null") != 0 r(conf, "testapp").provision_or_update() - assert os.system("getent passwd testapp &>/dev/null") == 0 - assert os.system("getent group testapp | grep -q sftp.app") != 0 + assert os.system("getent passwd testapp >/dev/null") == 0 + assert os.system("groups testapp | grep -q 'sftp.app'") != 0 conf["allow_sftp"] = True r(conf, "testapp").provision_or_update() - assert os.system("getent passwd testapp &>/dev/null") == 0 - assert os.system("getent group testapp | grep -q sftp.app") == 0 + assert os.system("getent passwd testapp >/dev/null") == 0 + assert os.system("groups testapp | grep -q 'sftp.app'") == 0 r(conf, "testapp").deprovision() - assert os.system("getent passwd testapp &>/dev/null") != 0 + assert os.system("getent passwd testapp 2>/dev/null") != 0 def test_resource_install_dir(): r = AppResourceClassesByType["install_dir"] - conf = {} + conf = {"owner": "nobody:rx", "group": "nogroup:rx"} # FIXME: should also check settings ? # FIXME: should also check automigrate from final_path @@ -159,10 +168,10 @@ def test_resource_install_dir(): assert os.path.exists("/var/www/testapp") unixperms = check_output("ls -ld /var/www/testapp").split() assert unixperms[0] == "dr-xr-x---" - assert unixperms[2] == "testapp" - assert unixperms[3] == "testapp" + assert unixperms[2] == "nobody" + assert unixperms[3] == "nogroup" - conf["group"] = "__APP__:rwx" + conf["owner"] = "nobody:rwx" conf["group"] = "www-data:x" r(conf, "testapp").provision_or_update() @@ -170,7 +179,7 @@ def test_resource_install_dir(): assert os.path.exists("/var/www/testapp") unixperms = check_output("ls -ld /var/www/testapp").split() assert unixperms[0] == "drwx--x---" - assert unixperms[2] == "testapp" + assert unixperms[2] == "nobody" assert unixperms[3] == "www-data" r(conf, "testapp").deprovision() @@ -181,7 +190,7 @@ def test_resource_install_dir(): def test_resource_data_dir(): r = AppResourceClassesByType["data_dir"] - conf = {} + conf = {"owner": "nobody:rx", "group": "nogroup:rx"} assert not os.path.exists("/home/yunohost.app/testapp") @@ -190,10 +199,10 @@ def test_resource_data_dir(): assert os.path.exists("/home/yunohost.app/testapp") unixperms = check_output("ls -ld /home/yunohost.app/testapp").split() assert unixperms[0] == "dr-xr-x---" - assert unixperms[2] == "testapp" - assert unixperms[3] == "testapp" + assert unixperms[2] == "nobody" + assert unixperms[3] == "nogroup" - conf["group"] = "__APP__:rwx" + conf["owner"] = "nobody:rwx" conf["group"] = "www-data:x" r(conf, "testapp").provision_or_update() @@ -201,12 +210,13 @@ def test_resource_data_dir(): assert os.path.exists("/home/yunohost.app/testapp") unixperms = check_output("ls -ld /home/yunohost.app/testapp").split() assert unixperms[0] == "drwx--x---" - assert unixperms[2] == "testapp" + assert unixperms[2] == "nobody" assert unixperms[3] == "www-data" r(conf, "testapp").deprovision() - assert not os.path.exists("/home/yunohost.app/testapp") + # FIXME : implement and check purge option + #assert not os.path.exists("/home/yunohost.app/testapp") def test_resource_port(): @@ -264,40 +274,42 @@ def test_resource_apt(): } } - assert os.system("dpkg --list | grep -q 'ii *nyancat'") != 0 - assert os.system("dpkg --list | grep -q 'ii *sl'") != 0 - assert os.system("dpkg --list | grep -q 'ii *yarn'") != 0 - assert os.system("dpkg --list | grep -q 'ii *lolcat'") != 0 - assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") != 0 + assert os.system("dpkg --list | grep -q 'ii *nyancat '") != 0 + assert os.system("dpkg --list | grep -q 'ii *sl '") != 0 + assert os.system("dpkg --list | grep -q 'ii *yarn '") != 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0 + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") != 0 r(conf, "testapp").provision_or_update() - assert os.system("dpkg --list | grep -q 'ii *nyancat'") == 0 - assert os.system("dpkg --list | grep -q 'ii *sl'") == 0 - assert os.system("dpkg --list | grep -q 'ii *yarn'") == 0 - assert os.system("dpkg --list | grep -q 'ii *lolcat'") != 0 # Lolcat shouldnt be installed yet - assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") == 0 + assert os.system("dpkg --list | grep -q 'ii *nyancat '") == 0 + assert os.system("dpkg --list | grep -q 'ii *sl '") == 0 + assert os.system("dpkg --list | grep -q 'ii *yarn '") == 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0 # Lolcat shouldnt be installed yet + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") == 0 conf["packages"] += ", lolcat" r(conf, "testapp").provision_or_update() - assert os.system("dpkg --list | grep -q 'ii *nyancat'") != 0 - assert os.system("dpkg --list | grep -q 'ii *sl'") != 0 - assert os.system("dpkg --list | grep -q 'ii *yarn'") != 0 - assert os.system("dpkg --list | grep -q 'ii *lolcat'") == 0 - assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") != 0 + assert os.system("dpkg --list | grep -q 'ii *nyancat '") == 0 + assert os.system("dpkg --list | grep -q 'ii *sl '") == 0 + assert os.system("dpkg --list | grep -q 'ii *yarn '") == 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat '") == 0 + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") == 0 r(conf, "testapp").deprovision() - assert os.system("dpkg --list | grep -q 'ii *nyancat'") != 0 - assert os.system("dpkg --list | grep -q 'ii *sl'") != 0 - assert os.system("dpkg --list | grep -q 'ii *yarn'") != 0 - assert os.system("dpkg --list | grep -q 'ii *lolcat'") != 0 - assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps'") != 0 + assert os.system("dpkg --list | grep -q 'ii *nyancat '") != 0 + assert os.system("dpkg --list | grep -q 'ii *sl '") != 0 + assert os.system("dpkg --list | grep -q 'ii *yarn '") != 0 + assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0 + assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") != 0 def test_resource_permissions(): + raise NotImplementedError() + r = AppResourceClassesByType["permissions"] conf = { "main": { diff --git a/src/utils/resources.py b/src/utils/resources.py index b4ec95bf8..1391e9c4a 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -216,7 +216,7 @@ class PermissionsResource(AppResource): super().__init__({"permissions": properties}, *args, **kwargs) - def provision_or_update(self, context: Dict): + def provision_or_update(self, context: Dict={}): from yunohost.permission import ( permission_create, @@ -267,7 +267,7 @@ class PermissionsResource(AppResource): permission_sync_to_user() - def deprovision(self, context: Dict): + def deprovision(self, context: Dict={}): from yunohost.permission import ( permission_delete, @@ -306,7 +306,7 @@ class SystemuserAppResource(AppResource): "allow_sftp": False } - def provision_or_update(self, context: Dict): + def provision_or_update(self, context: Dict={}): # FIXME : validate that no yunohost user exists with that name? # and/or that no system user exists during install ? @@ -328,7 +328,7 @@ class SystemuserAppResource(AppResource): os.system(f"usermod -G {','.join(groups)} {self.app}") - def deprovision(self, context: Dict): + def deprovision(self, context: Dict={}): if check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): os.system(f"deluser {self.app} >/dev/null") @@ -382,7 +382,7 @@ class InstalldirAppResource(AppResource): # FIXME: change default dir to /opt/stuff if app ain't a webapp ... - def provision_or_update(self, context: Dict): + def provision_or_update(self, context: Dict={}): current_install_dir = self.get_setting("install_dir") or self.get_setting("final_path") @@ -403,16 +403,17 @@ class InstalldirAppResource(AppResource): group, group_perm = self.group.split(":") owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0) group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0) - perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0" - chmod(self.dir, int(perm_octal)) + perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal + + chmod(self.dir, perm_octal) chown(self.dir, owner, group) # FIXME: shall we apply permissions recursively ? self.set_setting("install_dir", self.dir) self.delete_setting("final_path") # Legacy - def deprovision(self, context: Dict): + def deprovision(self, context: Dict={}): # FIXME : check that self.dir has a sensible value to prevent catastrophes if os.path.isdir(self.dir): rm(self.dir, recursive=True) @@ -444,7 +445,7 @@ class DatadirAppResource(AppResource): "group": "__APP__:rx", } - def provision_or_update(self, context: Dict): + def provision_or_update(self, context: Dict={}): current_data_dir = self.get_setting("data_dir") @@ -459,14 +460,14 @@ class DatadirAppResource(AppResource): group, group_perm = self.group.split(":") owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0) group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0) - perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0" + perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal - chmod(self.dir, int(perm_octal)) + chmod(self.dir, perm_octal) chown(self.dir, owner, group) self.set_setting("data_dir", self.dir) - def deprovision(self, context: Dict): + def deprovision(self, context: Dict={}): # FIXME: This should rm the datadir only if purge is enabled pass #if os.path.isdir(self.dir): @@ -497,7 +498,7 @@ class DatadirAppResource(AppResource): # "main": {"url": "?", "sha256sum": "?", "predownload": True} # } # -# def provision_or_update(self, context: Dict): +# def provision_or_update(self, context: Dict={}): # # FIXME # return # @@ -534,7 +535,7 @@ class AptDependenciesAppResource(AppResource): super().__init__(properties, *args, **kwargs) - def provision_or_update(self, context: Dict): + def provision_or_update(self, context: Dict={}): script = [f"ynh_install_app_dependencies {self.packages}"] for repo, values in self.extras.items(): @@ -542,7 +543,7 @@ class AptDependenciesAppResource(AppResource): self._run_script("provision_or_update", '\n'.join(script)) - def deprovision(self, context: Dict): + def deprovision(self, context: Dict={}): self._run_script("deprovision", "ynh_remove_app_dependencies") @@ -580,7 +581,7 @@ class PortResource(AppResource): cmd2 = f"grep --quiet \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml" return os.system(cmd1) == 0 and os.system(cmd2) == 0 - def provision_or_update(self, context: str): + def provision_or_update(self, context: Dict={}): # Don't do anything if port already defined ? if self.get_setting("port"): @@ -592,7 +593,7 @@ class PortResource(AppResource): self.set_setting("port", port) - def deprovision(self, context: Dict): + def deprovision(self, context: Dict={}): self.delete_setting("port") @@ -635,7 +636,7 @@ class DatabaseAppResource(AppResource): else: return False - def provision_or_update(self, context: str): + def provision_or_update(self, context: Dict={}): # This is equivalent to ynh_sanitize_dbid db_name = self.app.replace('-', '_').replace('.', '_') @@ -666,7 +667,7 @@ class DatabaseAppResource(AppResource): elif self.type == "postgresql": self._run_script("provision", f"ynh_psql_create_user '{db_user}' '{db_pwd}'; ynh_psql_create_db '{db_name}' '{db_user}'") - def deprovision(self, context: Dict): + def deprovision(self, context: Dict={}): db_name = self.app.replace('-', '_').replace('.', '_') db_user = db_name @@ -676,7 +677,9 @@ class DatabaseAppResource(AppResource): elif self.type == "postgresql": self._run_script("deprovision", f"ynh_psql_remove_db '{db_name}' '{db_user}'") - # FIXME : in fact we should delete settings to be consistent + self.delete_setting("db_name") + self.delete_setting("db_user") + self.delete_setting("db_pwd") AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()} From 9ed5b66bb403874c3b8f18b84900950e6680414d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 22 Jan 2022 16:21:47 +0100 Subject: [PATCH 23/54] Fixes /o\ --- src/migrations/0021_migrate_to_bullseye.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/migrations/0021_migrate_to_bullseye.py b/src/migrations/0021_migrate_to_bullseye.py index f4361cb19..b2abbb026 100644 --- a/src/migrations/0021_migrate_to_bullseye.py +++ b/src/migrations/0021_migrate_to_bullseye.py @@ -15,8 +15,8 @@ from yunohost.tools import ( ) from yunohost.app import unstable_apps from yunohost.regenconf import manually_modified_files, _force_clear_hashes -from yunohost.utils.filesystem import free_space_in_directory -from yunohost.utils.packages import ( +from yunohost.utils.system import ( + free_space_in_directory, get_ynh_package_version, _list_upgradable_apt_packages, ) From d9873e085d5327147633525ca3485a0993cbd235 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jan 2022 21:21:41 +0100 Subject: [PATCH 24/54] app resources: Support several ports --- src/tests/test_app_resources.py | 23 ++++++++++++-- src/utils/resources.py | 55 +++++++++++++++++++++++++-------- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index 638f6b119..f53cf373e 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -219,9 +219,9 @@ def test_resource_data_dir(): #assert not os.path.exists("/home/yunohost.app/testapp") -def test_resource_port(): +def test_resource_ports(): - r = AppResourceClassesByType["port"] + r = AppResourceClassesByType["ports"] conf = {} assert not app_setting("testapp", "port") @@ -235,6 +235,25 @@ def test_resource_port(): assert not app_setting("testapp", "port") +def test_resource_ports_several(): + + r = AppResourceClassesByType["ports"] + conf = {"main.default": 12345, "foobar.default": 23456} + + assert not app_setting("testapp", "port") + assert not app_setting("testapp", "port_foobar") + + r(conf, "testapp").provision_or_update() + + assert app_setting("testapp", "port") + assert app_setting("testapp", "port_foobar") + + r(conf, "testapp").deprovision() + + assert not app_setting("testapp", "port") + assert not app_setting("testapp", "port_foobar") + + def test_resource_database(): r = AppResourceClassesByType["database"] diff --git a/src/utils/resources.py b/src/utils/resources.py index 1391e9c4a..b7d3c1694 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -21,6 +21,7 @@ import os import copy import shutil +import random from typing import Dict, Any from moulinette.utils.process import check_output @@ -548,7 +549,7 @@ class AptDependenciesAppResource(AppResource): self._run_script("deprovision", "ynh_remove_app_dependencies") -class PortResource(AppResource): +class PortsResource(AppResource): """ is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting) is_available -> true @@ -564,15 +565,32 @@ class PortResource(AppResource): restore -> nothing (restore port setting) """ - type = "port" + type = "ports" priority = 70 default_properties = { - "default": 5000, - "expose": False, # FIXME : implement logic for exposed port (allow/disallow in firewall ?) - # "protocol": "tcp", } + default_port_properties = { + "default": None, + "exposed": False, # or True(="Both"), "TCP", "UDP" # FIXME : implement logic for exposed port (allow/disallow in firewall ?) + "fixed": False, # FIXME: implement logic. Corresponding to wether or not the port is "fixed" or any random port is ok + } + + def __init__(self, properties: Dict[str, Any], *args, **kwargs): + + if "main" not in properties: + properties["main"] = {} + + for port, infos in properties.items(): + properties[port] = copy.copy(self.default_port_properties) + properties[port].update(infos) + + if properties[port]["default"] is None: + properties[port]["default"] = random.randint(10000, 60000) + + super().__init__({"ports": properties}, *args, **kwargs) + def _port_is_used(self, port): # FIXME : this could be less brutal than two os.system ... @@ -583,19 +601,30 @@ class PortResource(AppResource): def provision_or_update(self, context: Dict={}): - # Don't do anything if port already defined ? - if self.get_setting("port"): - return + for name, infos in self.ports.items(): - port = self.default - while self._port_is_used(port): - port += 1 + setting_name = f"port_{name}" if name != "main" else "port" + port_value = self.get_setting(setting_name) + if not port_value and name != "main": + # Automigrate from legacy setting foobar_port (instead of port_foobar) + legacy_setting_name = "{name}_port" + port_value = self.get_setting(legacy_setting_name) + self.set_setting(setting_name, port_value) + self.delete_setting(legacy_setting_name) + continue - self.set_setting("port", port) + if not port_value: + port_value = self.infos["default"] + while self._port_is_used(port_value): + port_value += 1 + + self.set_setting(setting_name, port_value) def deprovision(self, context: Dict={}): - self.delete_setting("port") + for name, infos in self.ports.items(): + setting_name = f"port_{name}" if name != "main" else "port" + self.delete_setting(setting_name) class DatabaseAppResource(AppResource): From 9cb97640b941867fde2b1fb0ca981913e915ea0e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 28 Jan 2022 22:12:01 +0100 Subject: [PATCH 25/54] manifestv2: misc fixes + add test for manifestv2 install --- src/app.py | 5 +++-- src/tests/test_apps.py | 35 ++++++++++++++++++++++++++++++++++- src/utils/resources.py | 7 ++++++- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/app.py b/src/app.py index f1abffd16..58ee7e017 100644 --- a/src/app.py +++ b/src/app.py @@ -1949,7 +1949,7 @@ def _convert_v1_manifest_to_v2(manifest): "ldap": "?", "sso": "?", "disk": "50M", - "ram": {"build": "50M", "runtime": "10M", "include_swap": False} + "ram": {"build": "50M", "runtime": "10M"} } maintainer = manifest.get("maintainer", {}).get("name") @@ -2301,7 +2301,8 @@ def _check_manifest_requirements(manifest: Dict, action: str): # Ram for build ram_build_requirement = manifest["integration"]["ram"]["build"] - ram_include_swap = manifest["integration"]["ram"]["include_swap"] + # Is "include_swap" really useful ? We should probably decide wether to always include it or not instead + ram_include_swap = manifest["integration"]["ram"].get("include_swap", False) ram, swap = ram_available() if ram_include_swap: diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 2a808b5bd..6ff2c5271 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -45,6 +45,7 @@ def clean(): "break_yo_system", "legacy_app", "legacy_app__2", + "manifestv2_app", "full_domain_app", "my_webapp", ] @@ -115,7 +116,10 @@ def app_expected_files(domain, app): if app.startswith("legacy_app"): yield "/var/www/%s/index.html" % app yield "/etc/yunohost/apps/%s/settings.yml" % app - yield "/etc/yunohost/apps/%s/manifest.json" % app + if "manifestv2" in app: + yield "/etc/yunohost/apps/%s/manifest.toml" % app + else: + yield "/etc/yunohost/apps/%s/manifest.json" % app yield "/etc/yunohost/apps/%s/scripts/install" % app yield "/etc/yunohost/apps/%s/scripts/remove" % app @@ -157,6 +161,15 @@ def install_legacy_app(domain, path, public=True): ) +def install_manifestv2_app(domain, path, public=True): + + app_install( + os.path.join(get_test_apps_dir(), "manivestv2_app_ynh"), + args="domain={}&path={}&init_main_permission={}".format(domain, path, "visitors" if public else "all_users"), + force=True, + ) + + def install_full_domain_app(domain): app_install( @@ -195,6 +208,26 @@ def test_legacy_app_install_main_domain(): assert app_is_not_installed(main_domain, "legacy_app") +def test_manifestv2_app_install_main_domain(): + + main_domain = _get_maindomain() + + install_manifestv2_app(main_domain, "/manifestv2") + + app_map_ = app_map(raw=True) + assert main_domain in app_map_ + assert "/manifestv2" in app_map_[main_domain] + assert "id" in app_map_[main_domain]["/manifestv2"] + assert app_map_[main_domain]["/manifestv2"]["id"] == "manifestv2_app" + + assert app_is_installed(main_domain, "manifestv2_app") + assert app_is_exposed_on_http(main_domain, "/manifestv2", "Hextris") + + app_remove("manifestv2_app") + + assert app_is_not_installed(main_domain, "manifestv2_app") + + def test_app_from_catalog(): main_domain = _get_maindomain() diff --git a/src/utils/resources.py b/src/utils/resources.py index b7d3c1694..f64ad6134 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -45,6 +45,11 @@ class AppResourceManager: self.current = current self.wanted = wanted + if "resources" not in self.current: + self.current["resources"] = {} + if "resources" not in self.wanted: + self.wanted["resources"] = {} + def apply(self, rollback_if_failure, **context): todos = list(self.compute_todos()) @@ -233,7 +238,7 @@ class PermissionsResource(AppResource): existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"] for perm in existing_perms: - if perm.split(".") not in self.permissions.keys(): + if perm.split(".")[0] not in self.permissions.keys(): permission_delete(perm, force=True, sync_perm=False) for perm, infos in self.permissions.items(): From 6316b94af4bade40c50012eddda128d7ab058a73 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 1 Feb 2022 14:30:15 +0100 Subject: [PATCH 26/54] helpers: missing ynh_requirement definition --- helpers/utils | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helpers/utils b/helpers/utils index c1c8b9fc6..e40d4dd8a 100644 --- a/helpers/utils +++ b/helpers/utils @@ -927,6 +927,8 @@ ynh_compare_current_package_version() { _ynh_apply_default_permissions() { local target=$1 + local ynh_requirement=$(ynh_read_manifest --key=".requirements.yunohost") + if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then chmod o-rwx $target chmod g-w $target From 05a459d4df6cd2b10d6ea5e9e13882fdfe80b1ef Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 1 Feb 2022 14:59:24 +0100 Subject: [PATCH 27/54] Fix tests --- src/tests/test_questions.py | 651 +++++++++++++++--------------------- src/utils/config.py | 1 + 2 files changed, 277 insertions(+), 375 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 5917d32d4..e49047469 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -23,37 +23,38 @@ from yunohost.utils.error import YunohostError, YunohostValidationError """ Argument default format: { - "name": "the_name", - "type": "one_of_the_available_type", // "sting" is not specified - "ask": { - "en": "the question in english", - "fr": "the question in french" - }, - "help": { - "en": "some help text in english", - "fr": "some help text in french" - }, - "example": "an example value", // optional - "default", "some stuff", // optional, not available for all types - "optional": true // optional, will skip if not answered + "the_name": { + "type": "one_of_the_available_type", // "sting" is not specified + "ask": { + "en": "the question in english", + "fr": "the question in french" + }, + "help": { + "en": "some help text in english", + "fr": "some help text in french" + }, + "example": "an example value", // optional + "default", "some stuff", // optional, not available for all types + "optional": true // optional, will skip if not answered + } } User answers: -{"name": "value", ...} +{"the_name": "value", ...} """ def test_question_empty(): - ask_questions_and_parse_answers([], {}) == [] + ask_questions_and_parse_answers({}, {}) == [] def test_question_string(): - questions = [ - { - "name": "some_string", + + questions = { + "some_string": { "type": "string", } - ] + } answers = {"some_string": "some_value"} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -65,12 +66,11 @@ def test_question_string(): def test_question_string_from_query_string(): - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "type": "string", } - ] + } answers = "foo=bar&some_string=some_value&lorem=ipsum" out = ask_questions_and_parse_answers(questions, answers)[0] @@ -81,11 +81,7 @@ def test_question_string_from_query_string(): def test_question_string_default_type(): - questions = [ - { - "name": "some_string", - } - ] + questions = {"some_string": {}} answers = {"some_string": "some_value"} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -96,11 +92,7 @@ def test_question_string_default_type(): def test_question_string_no_input(): - questions = [ - { - "name": "some_string", - } - ] + questions = {"some_string": {}} answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): @@ -108,12 +100,11 @@ def test_question_string_no_input(): def test_question_string_input(): - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": "some question", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -127,11 +118,7 @@ def test_question_string_input(): def test_question_string_input_no_ask(): - questions = [ - { - "name": "some_string", - } - ] + questions = {"some_string": {}} answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -145,12 +132,7 @@ def test_question_string_input_no_ask(): def test_question_string_no_input_optional(): - questions = [ - { - "name": "some_string", - "optional": True, - } - ] + questions = {"some_string": {"optional": True}} answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -161,13 +143,12 @@ def test_question_string_no_input_optional(): def test_question_string_optional_with_input(): - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": "some question", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -181,13 +162,12 @@ def test_question_string_optional_with_input(): def test_question_string_optional_with_empty_input(): - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": "some question", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value=""), patch.object( @@ -201,12 +181,11 @@ def test_question_string_optional_with_empty_input(): def test_question_string_optional_with_input_without_ask(): - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -220,13 +199,12 @@ def test_question_string_optional_with_input_without_ask(): def test_question_string_no_input_default(): - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": "some question", "default": "some_value", } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -238,12 +216,11 @@ def test_question_string_no_input_default(): def test_question_string_input_test_ask(): ask_text = "some question" - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": ask_text, } - ] + } answers = {} with patch.object( @@ -264,13 +241,12 @@ def test_question_string_input_test_ask(): def test_question_string_input_test_ask_with_default(): ask_text = "some question" default_text = "some example" - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": ask_text, "default": default_text, } - ] + } answers = {} with patch.object( @@ -292,13 +268,12 @@ def test_question_string_input_test_ask_with_default(): def test_question_string_input_test_ask_with_example(): ask_text = "some question" example_text = "some example" - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": ask_text, "example": example_text, } - ] + } answers = {} with patch.object( @@ -313,13 +288,12 @@ def test_question_string_input_test_ask_with_example(): def test_question_string_input_test_ask_with_help(): ask_text = "some question" help_text = "some_help" - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": ask_text, "help": help_text, } - ] + } answers = {} with patch.object( @@ -331,7 +305,7 @@ def test_question_string_input_test_ask_with_help(): def test_question_string_with_choice(): - questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] + questions = {"some_string": {"type": "string", "choices": ["fr", "en"]}} answers = {"some_string": "fr"} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -341,7 +315,7 @@ def test_question_string_with_choice(): def test_question_string_with_choice_prompt(): - questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] + questions = {"some_string": {"type": "string", "choices": ["fr", "en"]}} answers = {"some_string": "fr"} with patch.object(Moulinette, "prompt", return_value="fr"), patch.object( os, "isatty", return_value=True @@ -354,7 +328,7 @@ def test_question_string_with_choice_prompt(): def test_question_string_with_choice_bad(): - questions = [{"name": "some_string", "type": "string", "choices": ["fr", "en"]}] + questions = {"some_string": {"type": "string", "choices": ["fr", "en"]}} answers = {"some_string": "bad"} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): @@ -364,13 +338,12 @@ def test_question_string_with_choice_bad(): def test_question_string_with_choice_ask(): ask_text = "some question" choices = ["fr", "en", "es", "it", "ru"] - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "ask": ask_text, "choices": choices, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="ru") as prompt, patch.object( @@ -384,14 +357,13 @@ def test_question_string_with_choice_ask(): def test_question_string_with_choice_default(): - questions = [ - { - "name": "some_string", + questions = { + "some_string": { "type": "string", "choices": ["fr", "en"], "default": "en", } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -402,12 +374,11 @@ def test_question_string_with_choice_default(): def test_question_password(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", } - ] + } answers = {"some_password": "some_value"} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -417,12 +388,11 @@ def test_question_password(): def test_question_password_no_input(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", } - ] + } answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): @@ -430,13 +400,12 @@ def test_question_password_no_input(): def test_question_password_input(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": "some question", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -450,12 +419,11 @@ def test_question_password_input(): def test_question_password_input_no_ask(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -469,13 +437,12 @@ def test_question_password_input_no_ask(): def test_question_password_no_input_optional(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "optional": True, } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): @@ -485,9 +452,7 @@ def test_question_password_no_input_optional(): assert out.type == "password" assert out.value == "" - questions = [ - {"name": "some_password", "type": "password", "optional": True, "default": ""} - ] + questions = {"some_password": {"type": "password", "optional": True, "default": ""}} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -498,14 +463,13 @@ def test_question_password_no_input_optional(): def test_question_password_optional_with_input(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "ask": "some question", "type": "password", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -519,14 +483,13 @@ def test_question_password_optional_with_input(): def test_question_password_optional_with_empty_input(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "ask": "some question", "type": "password", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value=""), patch.object( @@ -540,13 +503,12 @@ def test_question_password_optional_with_empty_input(): def test_question_password_optional_with_input_without_ask(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( @@ -560,14 +522,13 @@ def test_question_password_optional_with_input_without_ask(): def test_question_password_no_input_default(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": "some question", "default": "some_value", } - ] + } answers = {} # no default for password! @@ -577,14 +538,13 @@ def test_question_password_no_input_default(): @pytest.mark.skip # this should raises def test_question_password_no_input_example(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": "some question", "example": "some_value", } - ] + } answers = {"some_password": "some_value"} # no example for password! @@ -594,13 +554,12 @@ def test_question_password_no_input_example(): def test_question_password_input_test_ask(): ask_text = "some question" - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": ask_text, } - ] + } answers = {} with patch.object( @@ -622,14 +581,13 @@ def test_question_password_input_test_ask(): def test_question_password_input_test_ask_with_example(): ask_text = "some question" example_text = "some example" - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": ask_text, "example": example_text, } - ] + } answers = {} with patch.object( @@ -644,14 +602,13 @@ def test_question_password_input_test_ask_with_example(): def test_question_password_input_test_ask_with_help(): ask_text = "some question" help_text = "some_help" - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": ask_text, "help": help_text, } - ] + } answers = {} with patch.object( @@ -663,14 +620,13 @@ def test_question_password_input_test_ask_with_help(): def test_question_password_bad_chars(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": "some question", "example": "some_value", } - ] + } for i in PasswordQuestion.forbidden_chars: with pytest.raises(YunohostError), patch.object( @@ -680,14 +636,13 @@ def test_question_password_bad_chars(): def test_question_password_strong_enough(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "type": "password", "ask": "some question", "example": "some_value", } - ] + } with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): # too short @@ -698,14 +653,13 @@ def test_question_password_strong_enough(): def test_question_password_optional_strong_enough(): - questions = [ - { - "name": "some_password", + questions = { + "some_password": { "ask": "some question", "type": "password", "optional": True, } - ] + } with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): # too short @@ -716,12 +670,11 @@ def test_question_password_optional_strong_enough(): def test_question_path(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", } - ] + } answers = {"some_path": "/some_value"} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -731,12 +684,11 @@ def test_question_path(): def test_question_path_no_input(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", } - ] + } answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): @@ -744,13 +696,12 @@ def test_question_path_no_input(): def test_question_path_input(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", "ask": "some question", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( @@ -764,12 +715,11 @@ def test_question_path_input(): def test_question_path_input_no_ask(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( @@ -783,13 +733,12 @@ def test_question_path_input_no_ask(): def test_question_path_no_input_optional(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", "optional": True, } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -800,14 +749,13 @@ def test_question_path_no_input_optional(): def test_question_path_optional_with_input(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "ask": "some question", "type": "path", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( @@ -821,14 +769,13 @@ def test_question_path_optional_with_input(): def test_question_path_optional_with_empty_input(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "ask": "some question", "type": "path", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value=""), patch.object( @@ -842,13 +789,12 @@ def test_question_path_optional_with_empty_input(): def test_question_path_optional_with_input_without_ask(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( @@ -862,14 +808,13 @@ def test_question_path_optional_with_input_without_ask(): def test_question_path_no_input_default(): - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "ask": "some question", "type": "path", "default": "some_value", } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -881,13 +826,12 @@ def test_question_path_no_input_default(): def test_question_path_input_test_ask(): ask_text = "some question" - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", "ask": ask_text, } - ] + } answers = {} with patch.object( @@ -908,14 +852,13 @@ def test_question_path_input_test_ask(): def test_question_path_input_test_ask_with_default(): ask_text = "some question" default_text = "someexample" - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", "ask": ask_text, "default": default_text, } - ] + } answers = {} with patch.object( @@ -937,14 +880,13 @@ def test_question_path_input_test_ask_with_default(): def test_question_path_input_test_ask_with_example(): ask_text = "some question" example_text = "some example" - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", "ask": ask_text, "example": example_text, } - ] + } answers = {} with patch.object( @@ -959,14 +901,13 @@ def test_question_path_input_test_ask_with_example(): def test_question_path_input_test_ask_with_help(): ask_text = "some question" help_text = "some_help" - questions = [ - { - "name": "some_path", + questions = { + "some_path": { "type": "path", "ask": ask_text, "help": help_text, } - ] + } answers = {} with patch.object( @@ -978,12 +919,11 @@ def test_question_path_input_test_ask_with_help(): def test_question_boolean(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", } - ] + } answers = {"some_boolean": "y"} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -993,12 +933,11 @@ def test_question_boolean(): def test_question_boolean_all_yes(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", } - ] + } for value in ["Y", "yes", "Yes", "YES", "1", 1, True, "True", "TRUE", "true"]: out = ask_questions_and_parse_answers(questions, {"some_boolean": value})[0] @@ -1008,12 +947,11 @@ def test_question_boolean_all_yes(): def test_question_boolean_all_no(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", } - ] + } for value in ["n", "N", "no", "No", "No", "0", 0, False, "False", "FALSE", "false"]: out = ask_questions_and_parse_answers(questions, {"some_boolean": value})[0] @@ -1024,12 +962,11 @@ def test_question_boolean_all_no(): # XXX apparently boolean are always False (0) by default, I'm not sure what to think about that def test_question_boolean_no_input(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): @@ -1039,12 +976,11 @@ def test_question_boolean_no_input(): def test_question_boolean_bad_input(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", } - ] + } answers = {"some_boolean": "stuff"} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): @@ -1052,13 +988,12 @@ def test_question_boolean_bad_input(): def test_question_boolean_input(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", "ask": "some question", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="y"), patch.object( @@ -1075,12 +1010,11 @@ def test_question_boolean_input(): def test_question_boolean_input_no_ask(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="y"), patch.object( @@ -1091,13 +1025,12 @@ def test_question_boolean_input_no_ask(): def test_question_boolean_no_input_optional(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", "optional": True, } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -1105,14 +1038,13 @@ def test_question_boolean_no_input_optional(): def test_question_boolean_optional_with_input(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "ask": "some question", "type": "boolean", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="y"), patch.object( @@ -1123,14 +1055,13 @@ def test_question_boolean_optional_with_input(): def test_question_boolean_optional_with_empty_input(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "ask": "some question", "type": "boolean", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value=""), patch.object( @@ -1142,13 +1073,12 @@ def test_question_boolean_optional_with_empty_input(): def test_question_boolean_optional_with_input_without_ask(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="n"), patch.object( @@ -1160,14 +1090,13 @@ def test_question_boolean_optional_with_input_without_ask(): def test_question_boolean_no_input_default(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "ask": "some question", "type": "boolean", "default": 0, } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): @@ -1177,14 +1106,13 @@ def test_question_boolean_no_input_default(): def test_question_boolean_bad_default(): - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "ask": "some question", "type": "boolean", "default": "bad default", } - ] + } answers = {} with pytest.raises(YunohostError): ask_questions_and_parse_answers(questions, answers) @@ -1192,13 +1120,12 @@ def test_question_boolean_bad_default(): def test_question_boolean_input_test_ask(): ask_text = "some question" - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", "ask": ask_text, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value=0) as prompt, patch.object( @@ -1219,14 +1146,13 @@ def test_question_boolean_input_test_ask(): def test_question_boolean_input_test_ask_with_default(): ask_text = "some question" default_text = 1 - questions = [ - { - "name": "some_boolean", + questions = { + "some_boolean": { "type": "boolean", "ask": ask_text, "default": default_text, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value=1) as prompt, patch.object( @@ -1245,12 +1171,11 @@ def test_question_boolean_input_test_ask_with_default(): def test_question_domain_empty(): - questions = [ - { - "name": "some_domain", + questions = { + "some_domain": { "type": "domain", } - ] + } main_domain = "my_main_domain.com" answers = {} @@ -1271,12 +1196,11 @@ def test_question_domain_empty(): def test_question_domain(): main_domain = "my_main_domain.com" domains = [main_domain] - questions = [ - { - "name": "some_domain", + questions = { + "some_domain": { "type": "domain", } - ] + } answers = {"some_domain": main_domain} @@ -1295,12 +1219,11 @@ def test_question_domain_two_domains(): other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] - questions = [ - { - "name": "some_domain", + questions = { + "some_domain": { "type": "domain", } - ] + } answers = {"some_domain": other_domain} with patch.object( @@ -1329,12 +1252,11 @@ def test_question_domain_two_domains_wrong_answer(): other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] - questions = [ - { - "name": "some_domain", + questions = { + "some_domain": { "type": "domain", } - ] + } answers = {"some_domain": "doesnt_exist.pouet"} with patch.object( @@ -1351,12 +1273,11 @@ def test_question_domain_two_domains_default_no_ask(): other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] - questions = [ - { - "name": "some_domain", + questions = { + "some_domain": { "type": "domain", } - ] + } answers = {} with patch.object( @@ -1378,7 +1299,7 @@ def test_question_domain_two_domains_default(): other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] - questions = [{"name": "some_domain", "type": "domain", "ask": "choose a domain"}] + questions = {"some_domain": {"type": "domain", "ask": "choose a domain"}} answers = {} with patch.object( @@ -1400,7 +1321,7 @@ def test_question_domain_two_domains_default_input(): other_domain = "some_other_domain.tld" domains = [main_domain, other_domain] - questions = [{"name": "some_domain", "type": "domain", "ask": "choose a domain"}] + questions = {"some_domain": {"type": "domain", "ask": "choose a domain"}} answers = {} with patch.object( @@ -1436,12 +1357,11 @@ def test_question_user_empty(): } } - questions = [ - { - "name": "some_user", + questions = { + "some_user": { "type": "user", } - ] + } answers = {} with patch.object(user, "user_list", return_value={"users": users}): @@ -1463,12 +1383,11 @@ def test_question_user(): } } - questions = [ - { - "name": "some_user", + questions = { + "some_user": { "type": "user", } - ] + } answers = {"some_user": username} with patch.object(user, "user_list", return_value={"users": users}), patch.object( @@ -1501,12 +1420,11 @@ def test_question_user_two_users(): }, } - questions = [ - { - "name": "some_user", + questions = { + "some_user": { "type": "user", } - ] + } answers = {"some_user": other_user} with patch.object(user, "user_list", return_value={"users": users}), patch.object( @@ -1550,12 +1468,11 @@ def test_question_user_two_users_wrong_answer(): }, } - questions = [ - { - "name": "some_user", + questions = { + "some_user": { "type": "user", } - ] + } answers = {"some_user": "doesnt_exist.pouet"} with patch.object(user, "user_list", return_value={"users": users}): @@ -1585,7 +1502,7 @@ def test_question_user_two_users_no_default(): }, } - questions = [{"name": "some_user", "type": "user", "ask": "choose a user"}] + questions = {"some_user": {"type": "user", "ask": "choose a user"}} answers = {} with patch.object(user, "user_list", return_value={"users": users}): @@ -1615,7 +1532,7 @@ def test_question_user_two_users_default_input(): }, } - questions = [{"name": "some_user", "type": "user", "ask": "choose a user"}] + questions = {"some_user": {"type": "user", "ask": "choose a user"}} answers = {} with patch.object(user, "user_list", return_value={"users": users}), patch.object( @@ -1639,12 +1556,11 @@ def test_question_user_two_users_default_input(): def test_question_number(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", } - ] + } answers = {"some_number": 1337} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -1654,12 +1570,11 @@ def test_question_number(): def test_question_number_no_input(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", } - ] + } answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): @@ -1667,12 +1582,11 @@ def test_question_number_no_input(): def test_question_number_bad_input(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", } - ] + } answers = {"some_number": "stuff"} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): @@ -1684,13 +1598,12 @@ def test_question_number_bad_input(): def test_question_number_input(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", "ask": "some question", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( @@ -1722,12 +1635,11 @@ def test_question_number_input(): def test_question_number_input_no_ask(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( @@ -1741,13 +1653,12 @@ def test_question_number_input_no_ask(): def test_question_number_no_input_optional(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", "optional": True, } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -1758,14 +1669,13 @@ def test_question_number_no_input_optional(): def test_question_number_optional_with_input(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "ask": "some question", "type": "number", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( @@ -1779,13 +1689,12 @@ def test_question_number_optional_with_input(): def test_question_number_optional_with_input_without_ask(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", "optional": True, } - ] + } answers = {} with patch.object(Moulinette, "prompt", return_value="0"), patch.object( @@ -1799,14 +1708,13 @@ def test_question_number_optional_with_input_without_ask(): def test_question_number_no_input_default(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "ask": "some question", "type": "number", "default": 1337, } - ] + } answers = {} with patch.object(os, "isatty", return_value=False): out = ask_questions_and_parse_answers(questions, answers)[0] @@ -1817,14 +1725,13 @@ def test_question_number_no_input_default(): def test_question_number_bad_default(): - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "ask": "some question", "type": "number", "default": "bad default", } - ] + } answers = {} with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): ask_questions_and_parse_answers(questions, answers) @@ -1832,13 +1739,12 @@ def test_question_number_bad_default(): def test_question_number_input_test_ask(): ask_text = "some question" - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", "ask": ask_text, } - ] + } answers = {} with patch.object( @@ -1859,14 +1765,13 @@ def test_question_number_input_test_ask(): def test_question_number_input_test_ask_with_default(): ask_text = "some question" default_value = 1337 - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", "ask": ask_text, "default": default_value, } - ] + } answers = {} with patch.object( @@ -1888,14 +1793,13 @@ def test_question_number_input_test_ask_with_default(): def test_question_number_input_test_ask_with_example(): ask_text = "some question" example_value = 1337 - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", "ask": ask_text, "example": example_value, } - ] + } answers = {} with patch.object( @@ -1910,14 +1814,13 @@ def test_question_number_input_test_ask_with_example(): def test_question_number_input_test_ask_with_help(): ask_text = "some question" help_value = 1337 - questions = [ - { - "name": "some_number", + questions = { + "some_number": { "type": "number", "ask": ask_text, "help": help_value, } - ] + } answers = {} with patch.object( @@ -1929,7 +1832,7 @@ def test_question_number_input_test_ask_with_help(): def test_question_display_text(): - questions = [{"name": "some_app", "type": "display_text", "ask": "foobar"}] + questions = {"some_app": {"type": "display_text", "ask": "foobar"}} answers = {} with patch.object(sys, "stdout", new_callable=StringIO) as stdout, patch.object( @@ -1947,12 +1850,11 @@ def test_question_file_from_cli(): os.system(f"rm -f {filename}") os.system(f"echo helloworld > {filename}") - questions = [ - { - "name": "some_file", + questions = { + "some_file": { "type": "file", } - ] + } answers = {"some_file": filename} out = ask_questions_and_parse_answers(questions, answers)[0] @@ -1978,12 +1880,11 @@ def test_question_file_from_api(): from base64 import b64encode b64content = b64encode(b"helloworld") - questions = [ - { - "name": "some_file", + questions = { + "some_file": { "type": "file", } - ] + } answers = {"some_file": b64content} interface_type_bkp = Moulinette.interface.type diff --git a/src/utils/config.py b/src/utils/config.py index 9ae75969b..eb5f9b683 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -1408,6 +1408,7 @@ def ask_questions_and_parse_answers( out = [] for name, raw_question in raw_questions.items(): + raw_question['name'] = name question_class = ARGUMENTS_TYPE_PARSERS[raw_question.get("type", "string")] raw_question["value"] = answers.get(name) question = question_class(raw_question, context=context, hooks=hooks) From d8a924353226360de5466f2b605d49e66c99d93e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 1 Feb 2022 15:26:45 +0100 Subject: [PATCH 28/54] =?UTF-8?q?Typo=20=C3=A9=5F=C3=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tests/test_apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 6ff2c5271..76397093d 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -164,7 +164,7 @@ def install_legacy_app(domain, path, public=True): def install_manifestv2_app(domain, path, public=True): app_install( - os.path.join(get_test_apps_dir(), "manivestv2_app_ynh"), + os.path.join(get_test_apps_dir(), "manifestv2_app_ynh"), args="domain={}&path={}&init_main_permission={}".format(domain, path, "visitors" if public else "all_users"), force=True, ) From e4863830c90e23c89c044b5f503219c5444f7a63 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 1 Feb 2022 15:32:34 +0100 Subject: [PATCH 29/54] Fix call to ask_questions_and_parse_answers --- src/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/config.py b/src/utils/config.py index eb5f9b683..aa6586baa 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -580,7 +580,7 @@ class ConfigPanel: prefilled_answers.update(self.new_values) questions = ask_questions_and_parse_answers( - section["options"], + {question["name"]: question for question in section["options"]}, prefilled_answers=prefilled_answers, current_values=self.values, hooks=self.hooks, From f9e15ff8a39b5a44efe61e374de2378cff277dc3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 1 Feb 2022 15:36:13 +0100 Subject: [PATCH 30/54] Add app-resources tests to ci --- .gitlab/ci/test.gitlab-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 27b9b4913..519ae427a 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -125,6 +125,15 @@ test-app-config: - src/app.py - src/utils/config.py +test-app-resources: + extends: .test-stage + script: + - python3 -m pytest src/tests/test_app_resources.py + only: + changes: + - src/app.py + - src/utils/resources.py + test-changeurl: extends: .test-stage script: From 61d7ba1e40178bb408e86b1c69afe11c03889183 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Apr 2022 22:06:50 +0200 Subject: [PATCH 31/54] manifestv2: fix ports resource test --- src/tests/test_app_resources.py | 2 +- src/utils/resources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index f53cf373e..820ad6cb7 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -238,7 +238,7 @@ def test_resource_ports(): def test_resource_ports_several(): r = AppResourceClassesByType["ports"] - conf = {"main.default": 12345, "foobar.default": 23456} + conf = {"main": {"default": 12345}, "foobar": {"default": 23456}} assert not app_setting("testapp", "port") assert not app_setting("testapp", "port_foobar") diff --git a/src/utils/resources.py b/src/utils/resources.py index f64ad6134..3d3d06cca 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -619,7 +619,7 @@ class PortsResource(AppResource): continue if not port_value: - port_value = self.infos["default"] + port_value = infos["default"] while self._port_is_used(port_value): port_value += 1 From d5fcc3828182cc09330be72437a37ab90e7fbd73 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Apr 2022 22:18:24 +0200 Subject: [PATCH 32/54] manifestv2: fix backup tests --- src/backup.py | 2 +- src/tests/test_backuprestore.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.py b/src/backup.py index a4be27eb2..bfada89e2 100644 --- a/src/backup.py +++ b/src/backup.py @@ -1513,7 +1513,7 @@ class RestoreManager: operation_logger.extra["env"] = env_dict operation_logger.flush() - manifest = _get_manifest_of_app(app_dir_in_archive) + manifest = _get_manifest_of_app(app_settings_in_archive) if manifest["packaging_format"] >= 2: from yunohost.utils.resources import AppResourceManager try: diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index 03c3aa0c7..17147f586 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -355,13 +355,13 @@ def test_backup_script_failure_handling(monkeypatch, mocker): @pytest.mark.with_backup_recommended_app_installed def test_backup_not_enough_free_space(monkeypatch, mocker): - def custom_disk_usage(path): + def custom_space_used_by_directory(path, *args, **kwargs): return 99999999999999999 def custom_free_space_in_directory(dirpath): return 0 - monkeypatch.setattr("yunohost.backup.disk_usage", custom_disk_usage) + monkeypatch.setattr("yunohost.backup.space_used_by_directory", custom_space_used_by_directory) monkeypatch.setattr( "yunohost.backup.free_space_in_directory", custom_free_space_in_directory ) From 3675daf26d26059650b015e798a648a55627858f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 6 May 2022 16:43:58 +0200 Subject: [PATCH 33/54] Fix broken test, having install questions is mandatory --- src/app.py | 1 + src/tests/test_permission.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/app.py b/src/app.py index d92e3c5ae..d8e0086a9 100644 --- a/src/app.py +++ b/src/app.py @@ -1984,6 +1984,7 @@ def _convert_v1_manifest_to_v2(manifest): manifest["maintainers"] = [maintainer] if maintainer else [] install_questions = manifest["arguments"]["install"] + manifest["install"] = {} for question in install_questions: name = question.pop("name") diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 4e7f9f53d..f2bff5507 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -78,6 +78,7 @@ def _permission_create_with_dummy_app( "name": app, "id": app, "description": {"en": "Dummy app to test permissions"}, + "arguments": {"install": []} }, f, ) From dd6c8976f877c289ffc7f785e5be265d9999c09c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 6 May 2022 21:14:44 +0200 Subject: [PATCH 34/54] manifestv2: drafty implementation for notifications/doc inside the manifest --- src/app.py | 51 +++++++++++++++++++++++++++++++++++++++++++++ src/utils/system.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index d8e0086a9..7d6efbc0d 100644 --- a/src/app.py +++ b/src/app.py @@ -23,6 +23,7 @@ Manage apps """ +import glob import os import toml import json @@ -174,6 +175,9 @@ def app_info(app, full=False, upgradable=False): ret["setting_path"] = setting_path ret["manifest"] = local_manifest + + # FIXME: maybe this is not needed ? default ask questions are + # already set during the _get_manifest_of_app earlier ? ret["manifest"]["install"] = _set_default_ask_questions( ret["manifest"].get("install", {}) ) @@ -181,6 +185,13 @@ def app_info(app, full=False, upgradable=False): ret["from_catalog"] = from_catalog + # Hydrate app notifications and doc + for pagename, pagecontent in ret["manifest"]["doc"].items(): + ret["manifest"]["doc"][pagename] = _hydrate_app_template(pagecontent, settings) + for step, notifications in ret["manifest"]["notifications"].items(): + for name, notification in notifications.items(): + notifications[name] = _hydrate_app_template(notification, settings) + ret["is_webapp"] = "domain" in settings and "path" in settings if ret["is_webapp"]: @@ -1954,9 +1965,49 @@ def _get_manifest_of_app(path): manifest = _convert_v1_manifest_to_v2(manifest) manifest["install"] = _set_default_ask_questions(manifest.get("install", {})) + manifest["doc"], manifest["notifications"] = _parse_app_doc_and_notifications(path) + return manifest +def _parse_app_doc_and_notifications(path): + + # FIXME: need to find a way to handle i18n in there (file named FOOBAR_fr.md etc.) + + doc = {} + + for pagename in glob.glob(os.path.join(path, "doc") + "/*.md"): + name = os.path.basename(pagename)[:-len('.md')] + doc[name] = read_file(pagename) + + notifications = {} + + for step in ["pre_install", "post_install", "pre_upgrade", "post_upgrade"]: + notifications[step] = {} + if os.path.exists(os.path.join(path, "doc", "notifications", f"{step}.md")): + notifications[step]["main"] = read_file(os.path.join(path, "doc", "notifications", f"{step}.md")) + else: + for notification in glob.glob(os.path.join(path, "doc", "notifications", f"{step}.d") + "/*.md"): + name = os.path.basename(notification)[:-len('.md')] + notifications[step][name].append(read_file(notification)) + + return doc, notifications + + +def _hydrate_app_template(template, data): + + stuff_to_replace = set(re.findall(r'__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__', template)) + + for stuff in stuff_to_replace: + + varname = stuff.strip("_").lower() + + if varname in data: + template = template.replace(stuff, data[varname]) + + return template + + def _convert_v1_manifest_to_v2(manifest): manifest = copy.deepcopy(manifest) diff --git a/src/utils/system.py b/src/utils/system.py index f7d37cf1c..2aaf0fa35 100644 --- a/src/utils/system.py +++ b/src/utils/system.py @@ -57,7 +57,7 @@ def space_used_by_directory(dirpath, follow_symlinks=True): return int(du_output.split()[0]) stat = os.statvfs(dirpath) - return stat.f_frsize * stat.f_blocks + return stat.f_frsize * stat.f_blocks # FIXME : this doesnt do what the function name suggest this does ... def human_to_binary(size: str) -> int: From ddd10f630c8a3dc20087f5b2a497e062b808c035 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 13 May 2022 22:14:20 +0200 Subject: [PATCH 35/54] manifestv2: implement test for permission ressource --- src/tests/test_app_resources.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index 820ad6cb7..3640adde0 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -4,7 +4,9 @@ import pytest from moulinette.utils.process import check_output from yunohost.app import app_setting +from yunohost.domain import _get_maindomain from yunohost.utils.resources import AppResource, AppResourceManager, AppResourceClassesByType +from yunohost.permission import user_permission_list, permission_delete dummyfile = "/tmp/dummyappresource-testapp" @@ -57,6 +59,10 @@ def clean(): os.system("apt remove lolcat sl nyancat yarn >/dev/null 2>/dev/null") os.system("userdel testapp 2>/dev/null") + for p in user_permission_list()["permissions"]: + if p.startswith("testapp."): + permission_delete(p, force=True, sync_perm=False) + def test_provision_dummy(): @@ -327,17 +333,35 @@ def test_resource_apt(): def test_resource_permissions(): - raise NotImplementedError() + maindomain = _get_maindomain() + os.system(f"echo 'domain: {maindomain}' >> /etc/yunohost/apps/testapp/settings.yml") + os.system("echo 'path: /testapp' >> /etc/yunohost/apps/testapp/settings.yml") + # A manager object is required to set the label of the app... + manager = AppResourceManager("testapp", current={}, wanted={"name": "Test App"}) r = AppResourceClassesByType["permissions"] conf = { "main": { "url": "/", "allowed": "visitors" # protected? + }, + "admin": { + "url": "/admin", + "allowed": "" } } - pass + res = user_permission_list(full=True)["permissions"] + assert not any(key.startswith("testapp.") for key in res) + r(conf, "testapp", manager).provision_or_update() + res = user_permission_list(full=True)["permissions"] + assert "testapp.main" in res + assert "visitors" in res["testapp.main"]["allowed"] + assert res["testapp.main"]["url"] == "/" + + assert "testapp.admin" in res + assert not res["testapp.admin"]["allowed"] + assert res["testapp.admin"]["url"] == "/admin" From 9902d191aa7cef5fc615697135fbc0932347c9f7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 14 May 2022 00:15:42 +0200 Subject: [PATCH 36/54] manifestv2: improve permission resouce test --- src/tests/test_app_resources.py | 35 ++++++++++++++++++++++++++++----- src/utils/resources.py | 2 +- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index 3640adde0..4f7651067 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -344,12 +344,8 @@ def test_resource_permissions(): "main": { "url": "/", "allowed": "visitors" - # protected? + # TODO: test protected? }, - "admin": { - "url": "/admin", - "allowed": "" - } } res = user_permission_list(full=True)["permissions"] @@ -361,7 +357,36 @@ def test_resource_permissions(): assert "testapp.main" in res assert "visitors" in res["testapp.main"]["allowed"] assert res["testapp.main"]["url"] == "/" + assert "testapp.admin" not in res + + conf["admin"] = { + "url": "/admin", + "allowed": "" + } + + r(conf, "testapp", manager).provision_or_update() + + res = user_permission_list(full=True)["permissions"] + + assert "testapp.main" in list(res.keys()) + assert "visitors" in res["testapp.main"]["allowed"] + assert res["testapp.main"]["url"] == "/" assert "testapp.admin" in res assert not res["testapp.admin"]["allowed"] assert res["testapp.admin"]["url"] == "/admin" + + conf["admin"]["url"] = "/adminpanel" + + r(conf, "testapp", manager).provision_or_update() + + res = user_permission_list(full=True)["permissions"] + + # FIXME FIXME FIXME : this is the current behavior but + # it is NOT okay. c.f. comment in the code + assert res["testapp.admin"]["url"] == "/admin" # should be '/adminpanel' + + r(conf, "testapp").deprovision() + + res = user_permission_list(full=True)["permissions"] + assert "testapp.main" not in res diff --git a/src/utils/resources.py b/src/utils/resources.py index 3d3d06cca..5bde70e83 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -238,7 +238,7 @@ class PermissionsResource(AppResource): existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"] for perm in existing_perms: - if perm.split(".")[0] not in self.permissions.keys(): + if perm.split(".")[1] not in self.permissions.keys(): permission_delete(perm, force=True, sync_perm=False) for perm, infos in self.permissions.items(): From 8b1333a83747d52c1c7478319b6334d78f4e8071 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 21 May 2022 14:23:08 +0200 Subject: [PATCH 37/54] manifestv2: iterate on notifications/doc + implement tests for it --- src/app.py | 10 +++--- src/app_catalog.py | 2 ++ src/tests/test_apps.py | 75 ++++++++++++++++++++++++++++++++++++++++++ src/utils/resources.py | 1 + 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/src/app.py b/src/app.py index 7d6efbc0d..0eb06d33b 100644 --- a/src/app.py +++ b/src/app.py @@ -760,8 +760,8 @@ def app_manifest(app): shutil.rmtree(extracted_app_folder) - raw_questions = manifest.get("arguments", {}).get("install", []) - manifest["arguments"]["install"] = hydrate_questions_with_choices(raw_questions) + raw_questions = manifest.get("install", {}).values() + manifest["install"] = hydrate_questions_with_choices(raw_questions) return manifest @@ -1978,18 +1978,18 @@ def _parse_app_doc_and_notifications(path): for pagename in glob.glob(os.path.join(path, "doc") + "/*.md"): name = os.path.basename(pagename)[:-len('.md')] - doc[name] = read_file(pagename) + doc[name] = read_file(pagename).strip() notifications = {} for step in ["pre_install", "post_install", "pre_upgrade", "post_upgrade"]: notifications[step] = {} if os.path.exists(os.path.join(path, "doc", "notifications", f"{step}.md")): - notifications[step]["main"] = read_file(os.path.join(path, "doc", "notifications", f"{step}.md")) + notifications[step]["main"] = read_file(os.path.join(path, "doc", "notifications", f"{step}.md")).strip() else: for notification in glob.glob(os.path.join(path, "doc", "notifications", f"{step}.d") + "/*.md"): name = os.path.basename(notification)[:-len('.md')] - notifications[step][name].append(read_file(notification)) + notifications[step][name].append(read_file(notification).strip()) return doc, notifications diff --git a/src/app_catalog.py b/src/app_catalog.py index 5244d4d81..d635f8ba4 100644 --- a/src/app_catalog.py +++ b/src/app_catalog.py @@ -232,6 +232,8 @@ def _load_apps_catalog(): ) continue + # FIXME: we may want to autoconvert all v0/v1 manifest to v2 here + # so that everything is consistent in terms of APIs, datastructure format etc info["repository"] = apps_catalog_id merged_catalog["apps"][app] = info diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 76397093d..2c6deac7d 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -15,6 +15,8 @@ from yunohost.app import ( _is_installed, app_upgrade, app_map, + app_manifest, + app_info, ) from yunohost.domain import _get_maindomain, domain_add, domain_remove, domain_list from yunohost.utils.error import YunohostError @@ -208,6 +210,32 @@ def test_legacy_app_install_main_domain(): assert app_is_not_installed(main_domain, "legacy_app") +def test_legacy_app_manifest_preinstall(): + + m = app_manifest(os.path.join(get_test_apps_dir(), "legacy_app_ynh")) + # v1 manifesto are expected to have been autoconverted to v2 + + assert "id" in m + assert "description" in m + assert "integration" in m + assert "install" in m + assert "doc" in m and not m["doc"] + assert "notifications" in m and not m["notifications"] + + +def test_manifestv2_app_manifest_preinstall(): + + m = app_manifest(os.path.join(get_test_apps_dir(), "manifestv2_app_ynh")) + + assert "id" in m + assert "install" in m + assert "description" in m + assert "doc" in m + assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"] + assert "notifications" in m + assert "This is a dummy disclaimer to display prior to the install" in m["notifications"]["pre_install"] + + def test_manifestv2_app_install_main_domain(): main_domain = _get_maindomain() @@ -228,6 +256,53 @@ def test_manifestv2_app_install_main_domain(): assert app_is_not_installed(main_domain, "manifestv2_app") +def test_manifestv2_app_info_postinstall(): + + main_domain = _get_maindomain() + install_manifestv2_app(main_domain, "/manifestv2") + m = app_info("manifestv2_app", full=True)["manifest"] + + assert "id" in m + assert "install" in m + assert "description" in m + assert "doc" in m + assert "The app install dir is /var/www/manifestv2_app" in m["doc"]["ADMIN"] + assert "notifications" in m + assert "The app install dir is /var/www/manifestv2_app" in m["notifications"]["post_install"] + assert "The app id is manifestv2_app" in m["notifications"]["post_install"] + assert f"The app url is {main_domain}/manifestv2" in m["notifications"]["post_install"] + + +def test_manifestv2_app_info_preupgrade(monkeypatch): + + from yunohost.app_catalog import _load_apps_catalog as original_load_apps_catalog + def custom_load_apps_catalog(*args, **kwargs): + + res = original_load_apps_catalog(*args, **kwargs) + res["apps"]["manifestv2_app"] = { + "id": "manifestv2_app", + "level": 10, + "lastUpdate": 999999999, + "maintained": True, + "manifest": app_manifest(os.path.join(get_test_apps_dir(), "manifestv2_app_ynh")), + } + res["apps"]["manifestv2_app"]["manifest"]["version"] = "99999~ynh1" + + return res + monkeypatch.setattr("yunohost.app._load_apps_catalog", custom_load_apps_catalog) + + main_domain = _get_maindomain() + install_manifestv2_app(main_domain, "/manifestv2") + i = app_info("manifestv2_app", full=True) + + assert i["upgradable"] == "yes" + assert i["new_version"] == "99999~ynh1" + # FIXME : as I write this test, I realize that this implies the catalog API + # does provide the notifications, which means the list builder script + # should parse the files in the original app repo, possibly with proper i18n etc + assert "This is a dummy disclaimer to display prior to any upgrade" \ + in i["from_catalog"]["manifest"]["notifications"]["pre_upgrade"] + def test_app_from_catalog(): main_domain = _get_maindomain() diff --git a/src/utils/resources.py b/src/utils/resources.py index 5bde70e83..46f2eba33 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -69,6 +69,7 @@ class AppResourceManager: elif todo == "update": logger.info(f"Updating {name} ...") new.provision_or_update(context=context) + # FIXME FIXME FIXME : this exception doesnt catch Ctrl+C ?!?! except Exception as e: exception = e # FIXME: better error handling ? display stacktrace ? From 2ccb0c8db6691a1d485d3a4be1f4fe197c5742b5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 21 May 2022 16:38:51 +0200 Subject: [PATCH 38/54] manifestv2: add some i18n support for doc/notifications --- src/app.py | 45 +++++++++++++++++++++++++++++++----------- src/tests/test_apps.py | 19 ++++++++++-------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/src/app.py b/src/app.py index 0eb06d33b..01d5cc7eb 100644 --- a/src/app.py +++ b/src/app.py @@ -1972,24 +1972,47 @@ def _get_manifest_of_app(path): def _parse_app_doc_and_notifications(path): - # FIXME: need to find a way to handle i18n in there (file named FOOBAR_fr.md etc.) - doc = {} - for pagename in glob.glob(os.path.join(path, "doc") + "/*.md"): - name = os.path.basename(pagename)[:-len('.md')] - doc[name] = read_file(pagename).strip() + for filepath in glob.glob(os.path.join(path, "doc") + "/*.md"): + + # to be improved : [a-z]{2,3} is a clumsy way of parsing the + # lang code ... some lang code are more complex that this é_è + m = re.match("([A-Z]*)(_[a-z]{2,3})?.md", filepath.split("/")[-1]) + + if not m: + # FIXME: shall we display a warning ? idk + continue + pagename, lang = m.groups() + lang = lang.strip("_") if lang else "en" + + if pagename not in doc: + doc[pagename] = {} + doc[pagename][lang] = read_file(filepath).strip() notifications = {} for step in ["pre_install", "post_install", "pre_upgrade", "post_upgrade"]: notifications[step] = {} - if os.path.exists(os.path.join(path, "doc", "notifications", f"{step}.md")): - notifications[step]["main"] = read_file(os.path.join(path, "doc", "notifications", f"{step}.md")).strip() - else: - for notification in glob.glob(os.path.join(path, "doc", "notifications", f"{step}.d") + "/*.md"): - name = os.path.basename(notification)[:-len('.md')] - notifications[step][name].append(read_file(notification).strip()) + for filepath in glob.glob(os.path.join(path, "doc", "notifications", f"{step}*.md")): + m = re.match(step + "(_[a-z]{2,3})?.md", filepath.split("/")[-1]) + if not m: + continue + pagename = "main" + lang = m.groups()[0].strip("_") if m.groups()[0] else "en" + if pagename not in notifications[step]: + notifications[step][pagename] = {} + notifications[step][pagename][lang] = read_file(filepath).strip() + + for filepath in glob.glob(os.path.join(path, "doc", "notifications", f"{step}.d") + "/*.md"): + m = re.match(r"([A-Za-z0-9\.\~]*)(_[a-z]{2,3})?.md", filepath.split("/")[-1]) + if not m: + continue + pagename, lang = m.groups() + lang = lang.strip("_") if lang else "en" + if pagename not in notifications[step]: + notifications[step][pagename] = {} + notifications[step][pagename][lang] = read_file(filepath).strip() return doc, notifications diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 2c6deac7d..4661e54b2 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -219,8 +219,8 @@ def test_legacy_app_manifest_preinstall(): assert "description" in m assert "integration" in m assert "install" in m - assert "doc" in m and not m["doc"] - assert "notifications" in m and not m["notifications"] + assert m.get("doc") == {} + assert m.get("notifications") == {} def test_manifestv2_app_manifest_preinstall(): @@ -231,9 +231,11 @@ def test_manifestv2_app_manifest_preinstall(): assert "install" in m assert "description" in m assert "doc" in m - assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"] + assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"]["en"] + assert "Ceci est une fausse description des fonctionalités de l'app" in m["doc"]["DESCRIPTION"]["fr"] assert "notifications" in m - assert "This is a dummy disclaimer to display prior to the install" in m["notifications"]["pre_install"] + assert "This is a dummy disclaimer to display prior to the install" in m["notifications"]["pre_install"]["en"] + assert "Ceci est un faux disclaimer à présenter avant l'installation" in m["notifications"]["pre_install"]["fr"] def test_manifestv2_app_install_main_domain(): @@ -266,11 +268,12 @@ def test_manifestv2_app_info_postinstall(): assert "install" in m assert "description" in m assert "doc" in m - assert "The app install dir is /var/www/manifestv2_app" in m["doc"]["ADMIN"] + assert "The app install dir is /var/www/manifestv2_app" in m["doc"]["ADMIN"]["en"] + assert "Le dossier d'install de l'app est /var/www/manifestv2_app" in m["doc"]["ADMIN"]["fr"] assert "notifications" in m - assert "The app install dir is /var/www/manifestv2_app" in m["notifications"]["post_install"] - assert "The app id is manifestv2_app" in m["notifications"]["post_install"] - assert f"The app url is {main_domain}/manifestv2" in m["notifications"]["post_install"] + assert "The app install dir is /var/www/manifestv2_app" in m["notifications"]["post_install"]["en"] + assert "The app id is manifestv2_app" in m["notifications"]["post_install"]["en"] + assert f"The app url is {main_domain}/manifestv2" in m["notifications"]["post_install"]["en"] def test_manifestv2_app_info_preupgrade(monkeypatch): From d9e326f2cdf4643bd5aa5d4b39e2b497e2e21df7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 21 May 2022 17:11:41 +0200 Subject: [PATCH 39/54] manifestv2: print pre/post install notices during install in cli --- src/app.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/app.py b/src/app.py index 01d5cc7eb..720c65905 100644 --- a/src/app.py +++ b/src/app.py @@ -186,11 +186,13 @@ def app_info(app, full=False, upgradable=False): ret["from_catalog"] = from_catalog # Hydrate app notifications and doc - for pagename, pagecontent in ret["manifest"]["doc"].items(): - ret["manifest"]["doc"][pagename] = _hydrate_app_template(pagecontent, settings) + for pagename, content_per_lang in ret["manifest"]["doc"].items(): + for lang, content in content_per_lang.items(): + ret["manifest"]["doc"][pagename][lang] = _hydrate_app_template(content, settings) for step, notifications in ret["manifest"]["notifications"].items(): - for name, notification in notifications.items(): - notifications[name] = _hydrate_app_template(notification, settings) + for name, content_per_lang in notifications.items(): + for lang, content in content_per_lang.items(): + notifications[name][lang] = _hydrate_app_template(content, settings) ret["is_webapp"] = "domain" in settings and "path" in settings @@ -840,6 +842,15 @@ def app_install( _confirm_app_install(app, force) manifest, extracted_app_folder = _extract_app(app) + + # Display pre_install notices in cli mode + if manifest["notifications"]["pre_install"] and Moulinette.interface.type == "cli": + for notice in manifest["notifications"]["pre_install"].values(): + # Should we render the markdown maybe? idk + print("==========") + print(_value_for_locale(notice)) + print("==========") + packaging_format = manifest["packaging_format"] # Check ID @@ -1090,6 +1101,17 @@ def app_install( logger.success(m18n.n("installation_complete")) + # Display post_install notices in cli mode + if manifest["notifications"]["post_install"] and Moulinette.interface.type == "cli": + # (Call app_info to get the version hydrated with settings) + infos = app_info(app_instance_name, full=True) + for notice in infos["manifest"]["notifications"]["post_install"].values(): + # Should we render the markdown maybe? idk + print("==========") + print(_value_for_locale(notice)) + print("==========") + + # Call postinstall hook hook_callback("post_app_install", env=env_dict) From dc1f5725d004075027fb745956a5113cb0342a10 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 21:47:02 +0200 Subject: [PATCH 40/54] manifestv2: fix v1/v2 conversion for maintainers --- src/app.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index 720c65905..b691b563b 100644 --- a/src/app.py +++ b/src/app.py @@ -2076,8 +2076,13 @@ def _convert_v1_manifest_to_v2(manifest): "ram": {"build": "50M", "runtime": "10M"} } - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] + maintainers = manifest.get("maintainer", {}) + if isinstance(maintainers, list): + maintainers = [m['name'] for m in maintainers] + else: + maintainers = [maintainers["name"]] if maintainers.get("name") else [] + + manifest["maintainers"] = maintainers install_questions = manifest["arguments"]["install"] From 0022abe896ef5025bbf5dadc3891af822f7bf170 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 22:11:17 +0200 Subject: [PATCH 41/54] manifestv2: switch to API v3 for catalog which includes v2 manifests --- src/app_catalog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app_catalog.py b/src/app_catalog.py index d635f8ba4..12bb4e6d7 100644 --- a/src/app_catalog.py +++ b/src/app_catalog.py @@ -19,7 +19,7 @@ logger = getActionLogger("yunohost.app_catalog") APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" -APPS_CATALOG_API_VERSION = 2 +APPS_CATALOG_API_VERSION = 3 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" From f4cb20f081a1e6cbec23d3770e9ef654a92e50c2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Sep 2022 16:19:38 +0200 Subject: [PATCH 42/54] manifestv2: try to fix a bunch of tests --- src/tests/test_apps.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 4661e54b2..4853817f9 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -220,7 +220,7 @@ def test_legacy_app_manifest_preinstall(): assert "integration" in m assert "install" in m assert m.get("doc") == {} - assert m.get("notifications") == {} + assert m.get["notifications"] == {"pre_install": {}, "pre_upgrade": {}, "post_install": {}, "post_upgrade": {}} def test_manifestv2_app_manifest_preinstall(): @@ -231,11 +231,11 @@ def test_manifestv2_app_manifest_preinstall(): assert "install" in m assert "description" in m assert "doc" in m - assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"]["en"] - assert "Ceci est une fausse description des fonctionalités de l'app" in m["doc"]["DESCRIPTION"]["fr"] + assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"]["main"]["en"] + assert "Ceci est une fausse description des fonctionalités de l'app" in m["doc"]["DESCRIPTION"]["main"]["fr"] assert "notifications" in m - assert "This is a dummy disclaimer to display prior to the install" in m["notifications"]["pre_install"]["en"] - assert "Ceci est un faux disclaimer à présenter avant l'installation" in m["notifications"]["pre_install"]["fr"] + assert "This is a dummy disclaimer to display prior to the install" in m["notifications"]["pre_install"]["main"]["en"] + assert "Ceci est un faux disclaimer à présenter avant l'installation" in m["notifications"]["pre_install"]["main"]["fr"] def test_manifestv2_app_install_main_domain(): @@ -278,6 +278,8 @@ def test_manifestv2_app_info_postinstall(): def test_manifestv2_app_info_preupgrade(monkeypatch): + manifest = app_manifest(os.path.join(get_test_apps_dir(), "manifestv2_app_ynh")) + from yunohost.app_catalog import _load_apps_catalog as original_load_apps_catalog def custom_load_apps_catalog(*args, **kwargs): @@ -287,7 +289,7 @@ def test_manifestv2_app_info_preupgrade(monkeypatch): "level": 10, "lastUpdate": 999999999, "maintained": True, - "manifest": app_manifest(os.path.join(get_test_apps_dir(), "manifestv2_app_ynh")), + "manifest": manifest, } res["apps"]["manifestv2_app"]["manifest"]["version"] = "99999~ynh1" From 4faeabefa2f415afeaf95fbd6341a2f3f3b441ca Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Sep 2022 21:51:48 +0200 Subject: [PATCH 43/54] manifestv2: moar test fixes --- helpers/utils | 2 +- src/tests/test_apps.py | 13 +++++++------ src/utils/resources.py | 7 ++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/helpers/utils b/helpers/utils index 1b6e3cd50..7741ee02b 100644 --- a/helpers/utils +++ b/helpers/utils @@ -926,7 +926,7 @@ ynh_compare_current_package_version() { _ynh_apply_default_permissions() { local target=$1 - local ynh_requirement=$(ynh_read_manifest --key=".requirements.yunohost") + local ynh_requirement=$(ynh_read_manifest --manifest_key="requirements.yunohost") if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then chmod o-rwx $target diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 4853817f9..20a47b5d7 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -219,8 +219,8 @@ def test_legacy_app_manifest_preinstall(): assert "description" in m assert "integration" in m assert "install" in m - assert m.get("doc") == {} - assert m.get["notifications"] == {"pre_install": {}, "pre_upgrade": {}, "post_install": {}, "post_upgrade": {}} + assert m["doc"] == {} + assert m["notifications"] == {"pre_install": {}, "pre_upgrade": {}, "post_install": {}, "post_upgrade": {}} def test_manifestv2_app_manifest_preinstall(): @@ -271,9 +271,9 @@ def test_manifestv2_app_info_postinstall(): assert "The app install dir is /var/www/manifestv2_app" in m["doc"]["ADMIN"]["en"] assert "Le dossier d'install de l'app est /var/www/manifestv2_app" in m["doc"]["ADMIN"]["fr"] assert "notifications" in m - assert "The app install dir is /var/www/manifestv2_app" in m["notifications"]["post_install"]["en"] - assert "The app id is manifestv2_app" in m["notifications"]["post_install"]["en"] - assert f"The app url is {main_domain}/manifestv2" in m["notifications"]["post_install"]["en"] + assert "The app install dir is /var/www/manifestv2_app" in m["notifications"]["post_install"]["main"]["en"] + assert "The app id is manifestv2_app" in m["notifications"]["post_install"]["main"]["en"] + assert f"The app url is {main_domain}/manifestv2" in m["notifications"]["post_install"]["main"]["en"] def test_manifestv2_app_info_preupgrade(monkeypatch): @@ -290,6 +290,7 @@ def test_manifestv2_app_info_preupgrade(monkeypatch): "lastUpdate": 999999999, "maintained": True, "manifest": manifest, + "state": "working", } res["apps"]["manifestv2_app"]["manifest"]["version"] = "99999~ynh1" @@ -306,7 +307,7 @@ def test_manifestv2_app_info_preupgrade(monkeypatch): # does provide the notifications, which means the list builder script # should parse the files in the original app repo, possibly with proper i18n etc assert "This is a dummy disclaimer to display prior to any upgrade" \ - in i["from_catalog"]["manifest"]["notifications"]["pre_upgrade"] + in i["from_catalog"]["manifest"]["notifications"]["pre_upgrade"]["main"]["en"] def test_app_from_catalog(): main_domain = _get_maindomain() diff --git a/src/utils/resources.py b/src/utils/resources.py index 46f2eba33..f93543cb2 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -615,9 +615,10 @@ class PortsResource(AppResource): # Automigrate from legacy setting foobar_port (instead of port_foobar) legacy_setting_name = "{name}_port" port_value = self.get_setting(legacy_setting_name) - self.set_setting(setting_name, port_value) - self.delete_setting(legacy_setting_name) - continue + if port_value: + self.set_setting(setting_name, port_value) + self.delete_setting(legacy_setting_name) + continue if not port_value: port_value = infos["default"] From e9190a6bd6f9f1fe3d55306fb798539e5f4b2989 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Sep 2022 23:55:16 +0200 Subject: [PATCH 44/54] manifestv2: moar test fixes --- src/tests/test_apps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 20a47b5d7..e62680824 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -231,8 +231,8 @@ def test_manifestv2_app_manifest_preinstall(): assert "install" in m assert "description" in m assert "doc" in m - assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"]["main"]["en"] - assert "Ceci est une fausse description des fonctionalités de l'app" in m["doc"]["DESCRIPTION"]["main"]["fr"] + assert "This is a dummy description of this app features" in m["doc"]["DESCRIPTION"]["en"] + assert "Ceci est une fausse description des fonctionalités de l'app" in m["doc"]["DESCRIPTION"]["fr"] assert "notifications" in m assert "This is a dummy disclaimer to display prior to the install" in m["notifications"]["pre_install"]["main"]["en"] assert "Ceci est un faux disclaimer à présenter avant l'installation" in m["notifications"]["pre_install"]["main"]["fr"] From 0610a1808b12b218651cf9e1b7faf3431882d568 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 2 Sep 2022 00:20:44 +0200 Subject: [PATCH 45/54] =?UTF-8?q?manifestv2:=20attempt=20to=20fix=20mypy?= =?UTF-8?q?=20errors=20=C3=A9=5F=C3=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/resources.py | 14 ++++++++++++-- src/utils/system.py | 5 +++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index f93543cb2..9edad2970 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -22,7 +22,8 @@ import os import copy import shutil import random -from typing import Dict, Any +from typing import Optional, Dict, List, Union, Any, Mapping, Callable, no_type_check + from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger @@ -123,6 +124,8 @@ class AppResourceManager: class AppResource: + default_properties: Dict[str, Any] = None + def __init__(self, properties: Dict[str, Any], app: str, manager=None): self.app = app @@ -199,7 +202,7 @@ class PermissionsResource(AppResource): default_properties = { } - default_perm_properties = { + default_perm_properties: Dict[str, Any] = { "url": None, "additional_urls": [], "auth_header": True, @@ -208,6 +211,8 @@ class PermissionsResource(AppResource): "protected": False, } + permissions: Dict[str, Dict[str, Any]] = {} + def __init__(self, properties: Dict[str, Any], *args, **kwargs): # FIXME : if url != None, we should check that there's indeed a domain/path defined ? ie that app is a webapp @@ -362,6 +367,7 @@ class SystemuserAppResource(AppResource): # fi # +@no_type_check class InstalldirAppResource(AppResource): """ is_provisioned -> setting install_dir exists + /dir/ exists @@ -427,6 +433,7 @@ class InstalldirAppResource(AppResource): # FIXME : in fact we should delete settings to be consistent +@no_type_check class DatadirAppResource(AppResource): """ is_provisioned -> setting data_dir exists + /dir/ exists @@ -510,6 +517,7 @@ class DatadirAppResource(AppResource): # return # +@no_type_check class AptDependenciesAppResource(AppResource): """ is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn) @@ -583,6 +591,8 @@ class PortsResource(AppResource): "fixed": False, # FIXME: implement logic. Corresponding to wether or not the port is "fixed" or any random port is ok } + ports: Dict[str, Dict[str, Any]] + def __init__(self, properties: Dict[str, Any], *args, **kwargs): if "main" not in properties: diff --git a/src/utils/system.py b/src/utils/system.py index 2aaf0fa35..fe6d2d01b 100644 --- a/src/utils/system.py +++ b/src/utils/system.py @@ -21,6 +21,7 @@ import re import os import logging +from typing import Union from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError @@ -74,11 +75,11 @@ def human_to_binary(size: str) -> int: raise YunohostError(f"Invalid size suffix '{suffix}', expected one of {symbols}") try: - size = float(size) + size_ = float(size) except Exception: raise YunohostError(f"Failed to convert size {size} to float") - return size * factor[suffix] + return int(size_ * factor[suffix]) def binary_to_human(n: int) -> str: From 564c2de8153f2f7dbdbd3668e0898ddc83a178cc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 2 Sep 2022 20:34:41 +0200 Subject: [PATCH 46/54] manifestv2: moar attempts to fix mypy errors --- src/utils/resources.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 9edad2970..05499c9b7 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -22,8 +22,7 @@ import os import copy import shutil import random -from typing import Optional, Dict, List, Union, Any, Mapping, Callable, no_type_check - +from typing import Optional, Dict, List, Union, Any, Mapping, Callable from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger @@ -124,6 +123,7 @@ class AppResourceManager: class AppResource: + type: str = "" default_properties: Dict[str, Any] = None def __init__(self, properties: Dict[str, Any], app: str, manager=None): @@ -318,6 +318,9 @@ class SystemuserAppResource(AppResource): "allow_sftp": False } + allow_ssh: bool = False + allow_sftp: bool = False + def provision_or_update(self, context: Dict={}): # FIXME : validate that no yunohost user exists with that name? @@ -367,7 +370,6 @@ class SystemuserAppResource(AppResource): # fi # -@no_type_check class InstalldirAppResource(AppResource): """ is_provisioned -> setting install_dir exists + /dir/ exists @@ -393,10 +395,18 @@ class InstalldirAppResource(AppResource): "group": "__APP__:rx", } + dir: str = "" + owner: str = "" + group: str = "" + # FIXME: change default dir to /opt/stuff if app ain't a webapp ... def provision_or_update(self, context: Dict={}): + assert self.dir.strip() # Be paranoid about self.dir being empty... + assert self.owner.strip() + assert self.group.strip() + current_install_dir = self.get_setting("install_dir") or self.get_setting("final_path") # If during install, /var/www/$app already exists, assume that it's okay to remove and recreate it @@ -427,13 +437,17 @@ class InstalldirAppResource(AppResource): self.delete_setting("final_path") # Legacy def deprovision(self, context: Dict={}): + + assert self.dir.strip() # Be paranoid about self.dir being empty... + assert self.owner.strip() + assert self.group.strip() + # FIXME : check that self.dir has a sensible value to prevent catastrophes if os.path.isdir(self.dir): rm(self.dir, recursive=True) # FIXME : in fact we should delete settings to be consistent -@no_type_check class DatadirAppResource(AppResource): """ is_provisioned -> setting data_dir exists + /dir/ exists @@ -459,8 +473,16 @@ class DatadirAppResource(AppResource): "group": "__APP__:rx", } + dir: str = "" + owner: str = "" + group: str = "" + def provision_or_update(self, context: Dict={}): + assert self.dir.strip() # Be paranoid about self.dir being empty... + assert self.owner.strip() + assert self.group.strip() + current_data_dir = self.get_setting("data_dir") if not os.path.isdir(self.dir): @@ -482,6 +504,11 @@ class DatadirAppResource(AppResource): self.set_setting("data_dir", self.dir) def deprovision(self, context: Dict={}): + + assert self.dir.strip() # Be paranoid about self.dir being empty... + assert self.owner.strip() + assert self.group.strip() + # FIXME: This should rm the datadir only if purge is enabled pass #if os.path.isdir(self.dir): @@ -517,7 +544,6 @@ class DatadirAppResource(AppResource): # return # -@no_type_check class AptDependenciesAppResource(AppResource): """ is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn) @@ -542,6 +568,9 @@ class AptDependenciesAppResource(AppResource): "extras": {} } + packages: List = [] + extras: Dict[str: Any] = {} + def __init__(self, properties: Dict[str, Any], *args, **kwargs): for key, values in properties.get("extras", {}).items(): From 7bd5857b3cf38ea507e0e1d7f8a989dad068d8f2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 2 Sep 2022 21:01:39 +0200 Subject: [PATCH 47/54] manifestv2: fix some FIXME, add some others @_@ --- src/utils/resources.py | 43 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 05499c9b7..3db999c84 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -39,6 +39,9 @@ logger = getActionLogger("yunohost.app_resources") class AppResourceManager: + # FIXME : add some sort of documentation mechanism + # to create a have a detailed description of each resource behavior + def __init__(self, app: str, current: Dict, wanted: Dict): self.app = app @@ -156,6 +159,7 @@ class AppResource: def _run_script(self, action, script, env={}, user="root"): from yunohost.app import _make_tmp_workdir_for_app, _make_environment_for_app_script + from yunohost.hook import hook_exec_with_script_debug_if_failure tmpdir = _make_tmp_workdir_for_app(app=self.app) @@ -172,10 +176,29 @@ ynh_abort_if_errors write_to_file(script_path, script) - #print(env_) - - # FIXME : use the hook_exec_with_debug_instructions_stuff - ret, _ = hook_exec(script_path, env=env_) + from yunohost.log import OperationLogger + operation_logger = OperationLogger._instances[-1], # FIXME : this is an ugly hack :( + try: + ( + call_failed, + failure_message_with_debug_instructions, + ) = hook_exec_with_script_debug_if_failure( + script_path + env=env_, + operation_logger=operation_logger, + error_message_if_script_failed="An error occured inside the script snippet", + error_message_if_failed=lambda e: f"{action} failed for {self.type} : {e}" + ) + finally: + if call_failed: + raise YunohostError( + failure_message_with_debug_instructions, raw_msg=True + ) + else: + # FIXME: currently in app install code, we have + # more sophisticated code checking if this broke something on the system etc ... + # dunno if we want to do this here or manage it elsewhere + pass #print(ret) @@ -327,9 +350,10 @@ class SystemuserAppResource(AppResource): # and/or that no system user exists during install ? if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): - # FIXME: improve error handling ? + # FIXME: improve logging ? os.system wont log stdout / stderr cmd = f"useradd --system --user-group {self.app}" - os.system(cmd) + ret = os.system(cmd) + assert ret == 0, f"useradd command failed with exit code {ret}" if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip(): raise YunohostError(f"Failed to create system user for {self.app}", raw_msg=True) @@ -390,7 +414,7 @@ class InstalldirAppResource(AppResource): priority = 30 default_properties = { - "dir": "/var/www/__APP__", # FIXME or choose to move this elsewhere nowadays idk... + "dir": "/var/www/__APP__", "owner": "__APP__:rx", "group": "__APP__:rx", } @@ -417,7 +441,10 @@ class InstalldirAppResource(AppResource): if not os.path.isdir(self.dir): # Handle case where install location changed, in which case we shall move the existing install dir # FIXME: confirm that's what we wanna do + # Maybe a middle ground could be to compute the size, check that it's not too crazy (eg > 1G idk), + # and check for available space on the destination if current_install_dir and os.path.isdir(current_install_dir): + logger.warning(f"Moving {current_install_dir} to {self.dir} ... (this may take a while)") shutil.move(current_install_dir, self.dir) else: mkdir(self.dir) @@ -468,7 +495,7 @@ class DatadirAppResource(AppResource): priority = 40 default_properties = { - "dir": "/home/yunohost.app/__APP__", # FIXME or choose to move this elsewhere nowadays idk... + "dir": "/home/yunohost.app/__APP__", "owner": "__APP__:rx", "group": "__APP__:rx", } From ebe41411adc0ea23c79c663f12b02223064b0030 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 2 Sep 2022 21:03:25 +0200 Subject: [PATCH 48/54] Unused imports --- src/utils/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 3db999c84..22b1177f0 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -22,7 +22,7 @@ import os import copy import shutil import random -from typing import Optional, Dict, List, Union, Any, Mapping, Callable +from typing import Dict, Any from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger From 28d0b6d891197076382fc87b2f9f0232ca230f9e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 2 Sep 2022 21:04:00 +0200 Subject: [PATCH 49/54] Unused imports --- src/utils/system.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/system.py b/src/utils/system.py index fe6d2d01b..7e1d8746e 100644 --- a/src/utils/system.py +++ b/src/utils/system.py @@ -21,7 +21,6 @@ import re import os import logging -from typing import Union from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError From 74e745a291d651c2ea965a15931da7e875c3490c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 2 Sep 2022 21:44:00 +0200 Subject: [PATCH 50/54] Stupid typo >_> --- src/utils/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 22b1177f0..a7685083a 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -183,7 +183,7 @@ ynh_abort_if_errors call_failed, failure_message_with_debug_instructions, ) = hook_exec_with_script_debug_if_failure( - script_path + script_path, env=env_, operation_logger=operation_logger, error_message_if_script_failed="An error occured inside the script snippet", From 487ef303d86fa8543b1c0c4ea66739a45b2097b0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 3 Sep 2022 01:04:05 +0200 Subject: [PATCH 51/54] Typo again ugh --- src/utils/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index a7685083a..73284b5be 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -22,7 +22,7 @@ import os import copy import shutil import random -from typing import Dict, Any +from typing import Dict, Any, List from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger From f6970ba4034739cfd5c5a7d5584b0e392e03d08d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 3 Sep 2022 01:04:29 +0200 Subject: [PATCH 52/54] manifestv2: Add doc about each resource type + script to create doc file --- doc/generate_resource_doc.py | 12 ++ src/utils/resources.py | 312 ++++++++++++++++++++++++----------- 2 files changed, 224 insertions(+), 100 deletions(-) create mode 100644 doc/generate_resource_doc.py diff --git a/doc/generate_resource_doc.py b/doc/generate_resource_doc.py new file mode 100644 index 000000000..1e16a76d9 --- /dev/null +++ b/doc/generate_resource_doc.py @@ -0,0 +1,12 @@ +from yunohost.utils.resources import AppResourceClassesByType + +resources = sorted(AppResourceClassesByType.values(), key=lambda r: r.priority) + +for klass in resources: + + doc = klass.__doc__.replace("\n ", "\n") + + print("") + print(f"## {klass.type.replace('_', ' ').title()}") + print("") + print(doc) diff --git a/src/utils/resources.py b/src/utils/resources.py index 73284b5be..4bda6589f 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -205,22 +205,49 @@ ynh_abort_if_errors class PermissionsResource(AppResource): """ - is_provisioned -> main perm exists - is_available -> perm urls do not conflict + Configure the SSO permissions/tiles. Typically, webapps are expected to have a 'main' permission mapped to '/', meaning that a tile pointing to the `$domain/$path` will be available in the SSO for users allowed to access that app. - update -> refresh/update values for url/additional_urls/show_tile/auth/protected/... create new perms / delete any perm not listed - provision -> same as update? + Additional permissions can be created, typically to have a specific tile and/or access rules for the admin part of a webapp. - deprovision -> delete permissions + The list of allowed user/groups may be initialized using the content of the `init_{perm}_permission` question from the manifest, hence `init_main_permission` replaces the `is_public` question and shall contain a group name (typically, `all_users` or `visitors`). - deep_clean -> delete permissions for any __APP__.foobar where app not in app list... + ##### Example: + ```toml + [resources.permissions] + main.url = "/" + # (these two previous lines should be enough in the majority of cases) - backup -> handled elsewhere by the core, should be integrated in there (dump .ldif/yml?) - restore -> handled by the core, should be integrated in there (restore .ldif/yml?) + admin.url = "/admin" + admin.show_tile = false + admin.allowed = "admins" # Assuming the "admins" group exists (cf future developments ;)) + ``` + + ##### Properties (for each perm name): + - `url`: The relative URI corresponding to this permission. Typically `/` or `/something`. This property may be omitted for non-web permissions. + - `show_tile`: (default: `true` if `url` is defined) Wether or not a tile should be displayed for that permission in the user portal + - `allowed`: (default: nobody) The group initially allowed to access this perm, if `init_{perm}_permission` is not defined in the manifest questions. Note that the admin may tweak who is allowed/unallowed on that permission later on, this is only meant to **initialize** the permission. + - `auth_header`: (default: `true`) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true + - `protected`: (default: `false`) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. + - `additional_urls`: (default: none) List of additional URL for which access will be allowed/forbidden + + ##### Provision/Update: + - Delete any permissions that may exist and be related to this app yet is not declared anymore + - Loop over the declared permissions and create them if needed or update them with the new values (FIXME : update ain't implemented yet >_>) + + ##### Deprovision: + - Delete all permission related to this app + + ##### Legacy management: + - Legacy `is_public` setting will be deleted if it exists """ + # Notes for future ? + # deep_clean -> delete permissions for any __APP__.foobar where app not in app list... + # backup -> handled elsewhere by the core, should be integrated in there (dump .ldif/yml?) + # restore -> handled by the core, should be integrated in there (restore .ldif/yml?) + type = "permissions" - priority = 10 + priority = 80 default_properties = { } @@ -319,20 +346,33 @@ class PermissionsResource(AppResource): class SystemuserAppResource(AppResource): """ - is_provisioned -> user __APP__ exists - is_available -> user and group __APP__ doesn't exists + Provision a system user to be used by the app. The username is exactly equal to the app id - provision -> create user - update -> update values for home / shell / groups + ##### Example: + ```toml + [resources.system_user] + # (empty - defaults are usually okay) + ``` - deprovision -> delete user + ##### Properties: + - `allow_ssh`: (default: False) Adds the user to the ssh.app group, allowing SSH connection via this user + - `allow_sftp`: (defalt: False) Adds the user to the sftp.app group, allowing SFTP connection via this user - deep_clean -> uuuuh ? delete any user that could correspond to an app x_x ? + ##### Provision/Update: + - will create the system user if it doesn't exists yet + - will add/remove the ssh/sftp.app groups - backup -> nothing - restore -> provision + ##### Deprovision: + - deletes the user and group """ + # Notes for future? + # + # deep_clean -> uuuuh ? delete any user that could correspond to an app x_x ? + # + # backup -> nothing + # restore -> provision + type = "system_user" priority = 20 @@ -341,6 +381,9 @@ class SystemuserAppResource(AppResource): "allow_sftp": False } + # FIXME : wat do regarding ssl-cert, multimedia + # FIXME : wat do about home dir + allow_ssh: bool = False allow_sftp: bool = False @@ -362,8 +405,13 @@ class SystemuserAppResource(AppResource): if self.allow_ssh: groups.add("ssh.app") + elif "ssh.app" in groups: + groups.remove("ssh.app") + if self.allow_sftp: groups.add("sftp.app") + elif "sftp.app" in groups: + groups.remove("sftp.app") os.system(f"usermod -G {','.join(groups)} {self.app}") @@ -382,34 +430,42 @@ class SystemuserAppResource(AppResource): # FIXME : better logging and error handling, add stdout/stderr from the deluser/delgroup commands... -# # Check if the user exists on the system -#if os.system(f"getent passwd {self.username} &>/dev/null") != 0: -# if ynh_system_user_exists "$username"; then -# deluser $username -# fi -# # Check if the group exists on the system -#if os.system(f"getent group {self.username} &>/dev/null") != 0: -# if ynh_system_group_exists "$username"; then -# delgroup $username -# fi -# - class InstalldirAppResource(AppResource): """ - is_provisioned -> setting install_dir exists + /dir/ exists - is_available -> /dir/ doesn't exists + Creates a directory to be used by the app as the installation directory, typically where the app sources and assets are located. The corresponding path is stored in the settings as `install_dir` - provision -> create setting + create dir - update -> update perms ? + ##### Example: + ```toml + [resources.install_dir] + # (empty - defaults are usually okay) + ``` - deprovision -> delete dir + delete setting + ##### Properties: + - `dir`: (default: `/var/www/__APP__`) The full path of the install dir + - `owner`: (default: `__APP__:rx`) The owner (and owner permissions) for the install dir + - `group`: (default: `__APP__:rx`) The group (and group permissions) for the install dir - deep_clean -> uuuuh ? delete any dir in /var/www/ that would not correspond to an app x_x ? + ##### Provision/Update: + - during install, the folder will be deleted if it already exists (FIXME: is this what we want?) + - if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location + - otherwise, creates the directory if it doesn't exists yet + - (re-)apply permissions (only on the folder itself, not recursively) + - save the value of `dir` as `install_dir` in the app's settings, which can be then used by the app scripts (`$install_dir`) and conf templates (`__INSTALL_DIR__`) + + ##### Deprovision: + - recursively deletes the directory if it exists + + ##### Legacy management: + - In the past, the setting was called `final_path`. The code will automatically rename it as `install_dir`. + - As explained in the 'Provision/Update' section, the folder will also be moved if the location changed - backup -> cp install dir - restore -> cp install dir """ + # Notes for future? + # deep_clean -> uuuuh ? delete any dir in /var/www/ that would not correspond to an app x_x ? + # backup -> cp install dir + # restore -> cp install dir + type = "install_dir" priority = 30 @@ -477,20 +533,41 @@ class InstalldirAppResource(AppResource): class DatadirAppResource(AppResource): """ - is_provisioned -> setting data_dir exists + /dir/ exists - is_available -> /dir/ doesn't exists + Creates a directory to be used by the app as the data store directory, typically where the app multimedia or large assets added by users are located. The corresponding path is stored in the settings as `data_dir`. This resource behaves very similarly to install_dir. - provision -> create setting + create dir - update -> update perms ? + ##### Example: + ```toml + [resources.data_dir] + # (empty - defaults are usually okay) + ``` - deprovision -> (only if purge enabled...) delete dir + delete setting + ##### Properties: + - `dir`: (default: `/home/yunohost.app/__APP__`) The full path of the data dir + - `owner`: (default: `__APP__:rx`) The owner (and owner permissions) for the data dir + - `group`: (default: `__APP__:rx`) The group (and group permissions) for the data dir - deep_clean -> zblerg idk nothing + ##### Provision/Update: + - if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location + - otherwise, creates the directory if it doesn't exists yet + - (re-)apply permissions (only on the folder itself, not recursively) + - save the value of `dir` as `data_dir` in the app's settings, which can be then used by the app scripts (`$data_dir`) and conf templates (`__DATA_DIR__`) + + ##### Deprovision: + - recursively deletes the directory if it exists + - FIXME: this should only be done if the PURGE option is set + - FIXME: this should also delete the corresponding setting + + ##### Legacy management: + - In the past, the setting may have been called `datadir`. The code will automatically rename it as `data_dir`. + - As explained in the 'Provision/Update' section, the folder will also be moved if the location changed - backup -> cp data dir ? (if not backup_core_only) - restore -> cp data dir ? (if in backup) """ + # notes for future ? + # deep_clean -> zblerg idk nothing + # backup -> cp data dir ? (if not backup_core_only) + # restore -> cp data dir ? (if in backup) + type = "data_dir" priority = 40 @@ -510,10 +587,12 @@ class DatadirAppResource(AppResource): assert self.owner.strip() assert self.group.strip() - current_data_dir = self.get_setting("data_dir") + current_data_dir = self.get_setting("data_dir") or self.get_setting("datadir") if not os.path.isdir(self.dir): # Handle case where install location changed, in which case we shall move the existing install dir + # FIXME: same as install_dir, is this what we want ? + # FIXME: What if people manually mved the data dir and changed the setting value and dont want the folder to be moved ? x_x if current_data_dir and os.path.isdir(current_data_dir): shutil.move(current_data_dir, self.dir) else: @@ -529,6 +608,7 @@ class DatadirAppResource(AppResource): chown(self.dir, owner, group) self.set_setting("data_dir", self.dir) + self.delete_setting("datadir") # Legacy def deprovision(self, context: Dict={}): @@ -543,50 +623,37 @@ class DatadirAppResource(AppResource): # FIXME : in fact we should delete settings to be consistent -# -#class SourcesAppResource(AppResource): -# """ -# is_provisioned -> (if pre_download,) cache exists with appropriate checksum -# is_available -> curl HEAD returns 200 -# -# update -> none? -# provision -> full download + check checksum -# -# deprovision -> remove cache for __APP__ ? -# -# deep_clean -> remove all cache -# -# backup -> nothing -# restore -> nothing -# """ -# -# type = "sources" -# -# default_properties = { -# "main": {"url": "?", "sha256sum": "?", "predownload": True} -# } -# -# def provision_or_update(self, context: Dict={}): -# # FIXME -# return -# - class AptDependenciesAppResource(AppResource): """ - is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn) - is_available -> True? idk + Create a virtual package in apt, depending on the list of specified packages that the app needs. The virtual packages is called `$app-ynh-deps` (with `_` being replaced by `-` in the app name, see `ynh_install_app_dependencies`) - update -> update deps on __APP__-ynh-deps - provision -> create/update deps on __APP__-ynh-deps + ##### Example: + ```toml + [resources.apt] + packages = "nyancat, lolcat, sl" - deprovision -> remove __APP__-ynh-deps (+autoremove?) + # (this part is optional and corresponds to the legacy ynh_install_extra_app_dependencies helper) + extras.yarn.repo = "deb https://dl.yarnpkg.com/debian/ stable main" + extras.yarn.key = "https://dl.yarnpkg.com/debian/pubkey.gpg" + extras.yarn.packages = "yarn" + ``` - deep_clean -> remove any __APP__-ynh-deps for app not in app list + ##### Properties: + - `packages`: Comma-separated list of packages to be installed via `apt` + - `extras`: A dict of (repo, key, packages) corresponding to "extra" repositories to fetch dependencies from - backup -> nothing - restore = provision + ##### Provision/Update: + - The code literally calls the bash helpers `ynh_install_app_dependencies` and `ynh_install_extra_app_dependencies`, similar to what happens in v1. + + ##### Deprovision: + - The code literally calls the bash helper `ynh_remove_app_dependencies` """ + # Notes for future? + # deep_clean -> remove any __APP__-ynh-deps for app not in app list + # backup -> nothing + # restore = provision + type = "apt" priority = 50 @@ -596,7 +663,7 @@ class AptDependenciesAppResource(AppResource): } packages: List = [] - extras: Dict[str: Any] = {} + extras: Dict[str, Dict[str, str]] = {} def __init__(self, properties: Dict[str, Any], *args, **kwargs): @@ -611,6 +678,7 @@ class AptDependenciesAppResource(AppResource): script = [f"ynh_install_app_dependencies {self.packages}"] for repo, values in self.extras.items(): script += [f"ynh_install_extra_app_dependencies --repo='{values['repo']}' --key='{values['key']}' --package='{values['packages']}'"] + # FIXME : we're feeding the raw value of values['packages'] to the helper .. if we want to be consistent, may they should be comma-separated, though in the majority of cases, only a single package is installed from an extra repo.. self._run_script("provision_or_update", '\n'.join(script)) @@ -621,20 +689,44 @@ class AptDependenciesAppResource(AppResource): class PortsResource(AppResource): """ - is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting) - is_available -> true + Book port(s) to be used by the app, typically to be used to the internal reverse-proxy between nginx and the app process. - update -> true - provision -> find a port not used by any app + Note that because multiple ports can be booked, each properties is prefixed by the name of the port. `main` is a special name and will correspond to the setting `$port`, whereas for example `xmpp_client` will correspond to the setting `$port_xmpp_client`. - deprovision -> delete the port setting + ##### Example: + ```toml + [resources.port] + # (empty should be fine for most apps ... though you can customize stuff if absolutely needed) - deep_clean -> ? + main.default = 12345 # if you really want to specify a prefered value .. but shouldnt matter in the majority of cases - backup -> nothing (backup port setting) - restore -> nothing (restore port setting) + xmpp_client.default = 5222 # if you need another port, pick a name for it (here, "xmpp_client") + xmpp_client.exposed = "TCP" # here, we're telling that the port needs to be publicly exposed on TCP on the firewall + ``` + + ##### Properties (for every port name): + - `default`: The prefered value for the port. If this port is already being used by another process right now, or is booked in another app's setting, the code will increment the value until it finds a free port and store that value as the setting. If no value is specified, a random value between 10000 and 60000 is used. + - `exposed`: (default: `false`) Wether this port should be opened on the firewall and be publicly reachable. This should be kept to `false` for the majority of apps than only need a port for internal reverse-proxying! Possible values: `false`, `true`(=`Both`), `Both`, `TCP`, `UDP`. This will result in the port being opened on the firewall, and the diagnosis checking that a program answers on that port. (FIXME: this is not implemented yet) + - `fixed`: (default: `false`) Tells that the app absolutely needs the specific value provided in `default`, typically because it's needed for a specific protocol (FIXME: this is not implemented yet) + + ##### Provision/Update (for every port name): + - If not already booked, look for a free port, starting with the `default` value (or a random value between 10000 and 60000 if no `default` set) + - (FIXME) If `exposed` is not `false`, open the port in the firewall accordingly - otherwise make sure it's closed. + - The value of the port is stored in the `$port` setting for the `main` port, or `$port_NAME` for other `NAME`s + + ##### Deprovision: + - (FIXME) Close the ports on the firewall + - Deletes all the port settings + + ##### Legacy management: + - In the past, some settings may have been named `NAME_port` instead of `port_NAME`, in which case the code will automatically rename the old setting. """ + # Notes for future? + #deep_clean -> ? + #backup -> nothing (backup port setting) + #restore -> nothing (restore port setting) + type = "ports" priority = 70 @@ -702,24 +794,44 @@ class PortsResource(AppResource): class DatabaseAppResource(AppResource): """ - is_provisioned -> setting db_user, db_name, db_pwd exists - is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it) + Initialize a database, either using MySQL or Postgresql. Relevant DB infos are stored in settings `$db_name`, `$db_user` and `$db_pwd`. - provision -> setup the db + init the setting - update -> ?? + NB: only one DB can be handled in such a way (is there really an app that would need two completely different DB ?...) - deprovision -> delete the db + NB2: no automagic migration will happen in an suddenly change `type` from `mysql` to `postgresql` or viceversa in its life - deep_clean -> ... idk look into any db name that would not be related to any app ... + ##### Example: + ```toml + [resources.database] + type = "mysql" # or : "postgresql". Only these two values are supported + ``` - backup -> dump db - restore -> setup + inject db dump + ##### Properties: + - `type`: The database type, either `mysql` or `postgresql` + + ##### Provision/Update: + - (Re)set the `$db_name` and `$db_user` settings with the sanitized app name (replacing `-` and `.` with `_`) + - If `$db_pwd` doesn't already exists, pick a random database password and store it in that setting + - If the database doesn't exists yet, create the SQL user and DB using `ynh_mysql_create_db` or `ynh_psql_create_db`. + + ##### Deprovision: + - Drop the DB using `ynh_mysql_remove_db` or `ynh_psql_remove_db` + - Deletes the `db_name`, `db_user` and `db_pwd` settings + + ##### Legacy management: + - In the past, the sql passwords may have been named `mysqlpwd` or `psqlpwd`, in which case it will automatically be renamed as `db_pwd` """ + # Notes for future? + # deep_clean -> ... idk look into any db name that would not be related to any app ... + # backup -> dump db + # restore -> setup + inject db dump + type = "database" + priority = 90 default_properties = { - "type": None, + "type": None, # FIXME: eeeeeeeh is this really a good idea considering 'type' is supposed to be the resource type x_x } def __init__(self, properties: Dict[str, Any], *args, **kwargs): From 0d2cb690b3b20669df5725a96f4fe28f7c86f9d4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 3 Sep 2022 16:31:42 +0200 Subject: [PATCH 53/54] manifestv2: moar test fixes --- src/utils/resources.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 4bda6589f..9da0cedb7 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -32,7 +32,6 @@ from moulinette.utils.filesystem import ( ) from yunohost.utils.error import YunohostError -from yunohost.hook import hook_exec logger = getActionLogger("yunohost.app_resources") @@ -127,7 +126,7 @@ class AppResourceManager: class AppResource: type: str = "" - default_properties: Dict[str, Any] = None + default_properties: Dict[str, Any] = {} def __init__(self, properties: Dict[str, Any], app: str, manager=None): @@ -177,7 +176,14 @@ ynh_abort_if_errors write_to_file(script_path, script) from yunohost.log import OperationLogger - operation_logger = OperationLogger._instances[-1], # FIXME : this is an ugly hack :( + + if OperationLogger._instances: + # FIXME ? : this is an ugly hack :( + operation_logger = OperationLogger._instances[-1] + else: + operation_logger = OperationLogger("resource_snippet", [("app", self.app)], env=env_) + operation_logger.start() + try: ( call_failed, @@ -249,7 +255,7 @@ class PermissionsResource(AppResource): type = "permissions" priority = 80 - default_properties = { + default_properties: Dict[str, Any] = { } default_perm_properties: Dict[str, Any] = { @@ -376,7 +382,7 @@ class SystemuserAppResource(AppResource): type = "system_user" priority = 20 - default_properties = { + default_properties: Dict[str, Any] = { "allow_ssh": False, "allow_sftp": False } @@ -469,7 +475,7 @@ class InstalldirAppResource(AppResource): type = "install_dir" priority = 30 - default_properties = { + default_properties: Dict[str, Any] = { "dir": "/var/www/__APP__", "owner": "__APP__:rx", "group": "__APP__:rx", @@ -571,7 +577,7 @@ class DatadirAppResource(AppResource): type = "data_dir" priority = 40 - default_properties = { + default_properties: Dict[str, Any] = { "dir": "/home/yunohost.app/__APP__", "owner": "__APP__:rx", "group": "__APP__:rx", @@ -657,7 +663,7 @@ class AptDependenciesAppResource(AppResource): type = "apt" priority = 50 - default_properties = { + default_properties: Dict[str, Any] = { "packages": [], "extras": {} } @@ -730,7 +736,7 @@ class PortsResource(AppResource): type = "ports" priority = 70 - default_properties = { + default_properties: Dict[str, Any] = { } default_port_properties = { @@ -830,7 +836,7 @@ class DatabaseAppResource(AppResource): type = "database" priority = 90 - default_properties = { + default_properties: Dict[str, Any] = { "type": None, # FIXME: eeeeeeeh is this really a good idea considering 'type' is supposed to be the resource type x_x } From dd59a855541c75796aee3fa81f84548d782f4d42 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 4 Sep 2022 19:51:16 +0200 Subject: [PATCH 54/54] manifestv2: fix i18n --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 67d8feca5..2a50db16d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -44,7 +44,6 @@ "app_remove_after_failed_install": "Removing the app following the installation failure...", "app_removed": "{app} uninstalled", "app_requirements_checking": "Checking requirements for {app}...", - "app_requirements_unmeet": "Requirements are not met for {app}, the package {pkgname} ({version}) must be {spec}", "app_restore_failed": "Could not restore {app}: {error}", "app_restore_script_failed": "An error occured inside the app restore script", "app_sources_fetch_failed": "Could not fetch sources files, is the URL correct?", @@ -463,6 +462,7 @@ "log_regen_conf": "Regenerate system configurations '{}'", "log_remove_on_failed_install": "Remove '{}' after a failed installation", "log_remove_on_failed_restore": "Remove '{}' after a failed restore from a backup archive", + "log_resource_snippet": "Provisioning/deprovisioning/updating a resource", "log_selfsigned_cert_install": "Install self-signed certificate on '{}' domain", "log_tools_migrations_migrate_forward": "Run migrations", "log_tools_postinstall": "Postinstall your YunoHost server",