From 3dda3bc4d5793444e3585aa1655b616fa1d8b595 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Nov 2023 18:03:23 +0100 Subject: [PATCH] perf: improve perf for a bunch of operations by lazy import + lazy define of config-panel related stuff --- src/app.py | 176 ++++++++++++++-------------- src/domain.py | 316 ++++++++++++++++++++++++++------------------------ 2 files changed, 259 insertions(+), 233 deletions(-) diff --git a/src/app.py b/src/app.py index ddaa6bccb..d33b584e2 100644 --- a/src/app.py +++ b/src/app.py @@ -17,11 +17,11 @@ # along with this program. If not, see . # +import time import glob import os import shutil import yaml -import time import re import subprocess import tempfile @@ -45,13 +45,6 @@ from moulinette.utils.filesystem import ( chmod, ) -from yunohost.utils.configpanel import ConfigPanel -from yunohost.utils.form import ( - DomainOption, - WebPathOption, - ask_questions_and_parse_answers, - parse_raw_options, -) from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.system import ( @@ -414,6 +407,7 @@ def app_change_url(operation_logger, app, domain, path): path -- New path at which the application will be move """ + from yunohost.utils.form import DomainOption, WebPathOption from yunohost.hook import hook_exec_with_script_debug_if_failure, hook_callback from yunohost.service import service_reload_or_restart @@ -964,6 +958,8 @@ def app_upgrade( def app_manifest(app, with_screenshot=False): + from yunohost.utils.form import parse_raw_options + manifest, extracted_app_folder = _extract_app(app) manifest["install"] = parse_raw_options(manifest.get("install", {}), serialize=True) @@ -1060,6 +1056,7 @@ def app_install( ) from yunohost.regenconf import manually_modified_files from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers + from yunohost.utils.form import ask_questions_and_parse_answers # Check if disk space available if free_space_in_directory("/") <= 512 * 1000 * 1000: @@ -1393,7 +1390,7 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): permission_delete, permission_sync_to_user, ) - from yunohost.domain import domain_list, domain_config_get, domain_config_set + from yunohost.domain import domain_list, domain_config_set, _get_raw_domain_settings if not _is_installed(app): raise YunohostValidationError( @@ -1471,7 +1468,7 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): hook_remove(app) for domain in domain_list()["domains"]: - if domain_config_get(domain, "feature.app.default_app") == app: + if _get_raw_domain_settings(domain).get("default_app") == app: domain_config_set(domain, "feature.app.default_app", "_none") if ret == 0: @@ -1572,6 +1569,7 @@ def app_register_url(app, domain, path): domain -- The domain on which the app should be registered (e.g. your.domain.tld) path -- The path to be registered (e.g. /coffee) """ + from yunohost.utils.form import DomainOption, WebPathOption from yunohost.permission import ( permission_url, user_permission_update, @@ -1614,7 +1612,7 @@ def app_ssowatconf(): """ from yunohost.domain import ( domain_list, - domain_config_get, + _get_raw_domain_settings, _get_domain_portal_dict, ) from yunohost.permission import user_permission_list @@ -1654,8 +1652,8 @@ def app_ssowatconf(): # FIXME : this could be handled by nginx's regen conf to further simplify ssowat's code ... redirected_urls = {} for domain in domains: - default_app = domain_config_get(domain, "feature.app.default_app") - if default_app != "_none" and _is_installed(default_app): + default_app = _get_raw_domain_settings(domain).get("default_app") + if default_app not in ["_none", None] and _is_installed(default_app): app_settings = _get_app_settings(default_app) app_domain = app_settings["domain"] app_path = app_settings["path"] @@ -1753,11 +1751,13 @@ def app_change_label(app, new_label): def app_action_list(app): + AppConfigPanel = _get_AppConfigPanel() return AppConfigPanel(app).list_actions() @is_unit_operation() def app_action_run(operation_logger, app, action, args=None, args_file=None): + AppConfigPanel = _get_AppConfigPanel() return AppConfigPanel(app).run_action( action, args=args, args_file=args_file, operation_logger=operation_logger ) @@ -1779,6 +1779,7 @@ def app_config_get(app, key="", full=False, export=False): else: mode = "classic" + AppConfigPanel = _get_AppConfigPanel() try: config_ = AppConfigPanel(app) return config_.get(key, mode) @@ -1798,91 +1799,97 @@ def app_config_set( Apply a new app configuration """ + AppConfigPanel = _get_AppConfigPanel() config_ = AppConfigPanel(app) return config_.set(key, value, args, args_file, operation_logger=operation_logger) -class AppConfigPanel(ConfigPanel): - entity_type = "app" - save_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/settings.yml") - config_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/config_panel.toml") - settings_must_be_defined: bool = True +def _get_AppConfigPanel(): + from yunohost.utils.configpanel import ConfigPanel - def _get_raw_settings(self) -> "RawSettings": - return self._call_config_script("show") + class AppConfigPanel(ConfigPanel): + entity_type = "app" + save_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/settings.yml") + config_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/config_panel.toml") + settings_must_be_defined: bool = True - def _apply( - self, - form: "FormModel", - previous_settings: dict[str, Any], - exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, - ) -> None: - env = {key: str(value) for key, value in form.dict().items()} - return_content = self._call_config_script("apply", env=env) + def _get_raw_settings(self) -> "RawSettings": + return self._call_config_script("show") - # If the script returned validation error - # raise a ValidationError exception using - # the first key - errors = return_content.get("validation_errors") - if errors: - for key, message in errors.items(): - raise YunohostValidationError( - "app_argument_invalid", - name=key, - error=message, - ) + def _apply( + self, + form: "FormModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ) -> None: + env = {key: str(value) for key, value in form.dict().items()} + return_content = self._call_config_script("apply", env=env) - def _run_action(self, form: "FormModel", action_id: str) -> None: - env = {key: str(value) for key, value in form.dict().items()} - self._call_config_script(action_id, env=env) + # If the script returned validation error + # raise a ValidationError exception using + # the first key + errors = return_content.get("validation_errors") + if errors: + for key, message in errors.items(): + raise YunohostValidationError( + "app_argument_invalid", + name=key, + error=message, + ) - def _call_config_script( - self, action: str, env: Union[dict[str, Any], None] = None - ) -> dict[str, Any]: - from yunohost.hook import hook_exec + def _run_action(self, form: "FormModel", action_id: str) -> None: + env = {key: str(value) for key, value in form.dict().items()} + self._call_config_script(action_id, env=env) - if env is None: - env = {} + def _call_config_script( + self, action: str, env: Union[dict[str, Any], None] = None + ) -> dict[str, Any]: + from yunohost.hook import hook_exec - # Add default config script if needed - config_script = os.path.join( - APPS_SETTING_PATH, self.entity, "scripts", "config" - ) - if not os.path.exists(config_script): - logger.debug("Adding a default config script") - default_script = """#!/bin/bash -source /usr/share/yunohost/helpers -ynh_abort_if_errors -ynh_app_config_run $1 -""" - write_to_file(config_script, default_script) + if env is None: + env = {} - # Call config script to extract current values - logger.debug(f"Calling '{action}' action from config script") - app = self.entity - app_id, app_instance_nb = _parse_app_instance_name(app) - settings = _get_app_settings(app) - env.update( - { - "app_id": app_id, - "app": app, - "app_instance_nb": str(app_instance_nb), - "final_path": settings.get("final_path", ""), - "install_dir": settings.get("install_dir", ""), - "YNH_APP_BASEDIR": os.path.join(APPS_SETTING_PATH, app), - } - ) + # Add default config script if needed + config_script = os.path.join( + APPS_SETTING_PATH, self.entity, "scripts", "config" + ) + if not os.path.exists(config_script): + logger.debug("Adding a default config script") + default_script = """#!/bin/bash + source /usr/share/yunohost/helpers + ynh_abort_if_errors + ynh_app_config_run $1 + """ + write_to_file(config_script, default_script) - ret, values = hook_exec(config_script, args=[action], env=env) - if ret != 0: - if action == "show": - raise YunohostError("app_config_unable_to_read") - elif action == "apply": - raise YunohostError("app_config_unable_to_apply") - else: - raise YunohostError("app_action_failed", action=action, app=app) - return values + # Call config script to extract current values + logger.debug(f"Calling '{action}' action from config script") + app = self.entity + app_id, app_instance_nb = _parse_app_instance_name(app) + settings = _get_app_settings(app) + env.update( + { + "app_id": app_id, + "app": app, + "app_instance_nb": str(app_instance_nb), + "final_path": settings.get("final_path", ""), + "install_dir": settings.get("install_dir", ""), + "YNH_APP_BASEDIR": os.path.join(APPS_SETTING_PATH, app), + } + ) + + ret, values = hook_exec(config_script, args=[action], env=env) + if ret != 0: + if action == "show": + raise YunohostError("app_config_unable_to_read") + elif action == "apply": + raise YunohostError("app_config_unable_to_apply") + else: + raise YunohostError("app_action_failed", action=action, app=app) + return values + + return AppConfigPanel def _get_app_settings(app): @@ -2782,6 +2789,7 @@ def _get_conflicting_apps(domain, path, ignore_app=None): """ from yunohost.domain import _assert_domain_exists + from yunohost.utils.form import DomainOption, WebPathOption domain = DomainOption.normalize(domain) path = WebPathOption.normalize(path) diff --git a/src/domain.py b/src/domain.py index 12c4d966c..e1f0c5673 100644 --- a/src/domain.py +++ b/src/domain.py @@ -34,17 +34,8 @@ from moulinette.utils.filesystem import ( write_to_yaml, ) -from yunohost.app import ( - app_ssowatconf, - _installed_apps, - _get_app_settings, - _get_conflicting_apps, -) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf -from yunohost.utils.configpanel import ConfigPanel -from yunohost.utils.form import BaseOption from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.dns import is_yunohost_dyndns_domain from yunohost.log import is_unit_operation if TYPE_CHECKING: @@ -191,7 +182,7 @@ def domain_info(domain): domain -- Domain to be checked """ - from yunohost.app import app_info + from yunohost.app import app_info, _installed_apps, _get_app_settings from yunohost.dns import _get_registar_settings from yunohost.certificate import certificate_status @@ -268,6 +259,7 @@ def domain_add( from yunohost.utils.ldap import _get_ldap_interface from yunohost.utils.password import assert_password_is_strong_enough from yunohost.certificate import _certificate_install_selfsigned + from yunohost.utils.dns import is_yunohost_dyndns_domain if dyndns_recovery_password: operation_logger.data_to_redact.append(dyndns_recovery_password) @@ -383,8 +375,15 @@ def domain_remove( """ import glob from yunohost.hook import hook_callback - from yunohost.app import app_ssowatconf, app_info, app_remove + from yunohost.app import ( + app_ssowatconf, + app_info, + app_remove, + _get_app_settings, + _installed_apps, + ) from yunohost.utils.ldap import _get_ldap_interface + from yunohost.utils.dns import is_yunohost_dyndns_domain if dyndns_recovery_password: operation_logger.data_to_redact.append(dyndns_recovery_password) @@ -566,6 +565,7 @@ def domain_main_domain(operation_logger, new_main_domain=None): """ from yunohost.tools import _set_hostname + from yunohost.app import app_ssowatconf # If no new domain specified, we return the current main domain if not new_main_domain: @@ -614,6 +614,8 @@ def domain_url_available(domain, path): path -- The path to check (e.g. /coffee) """ + from yunohost.app import _get_conflicting_apps + return len(_get_conflicting_apps(domain, path)) == 0 @@ -623,7 +625,8 @@ def _get_raw_domain_settings(domain): so the file may be completely empty """ _assert_domain_exists(domain) - path = DomainConfigPanel.save_path_tpl.format(entity=domain) + # NB: this corresponds to save_path_tpl in DomainConfigPanel + path = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" if os.path.exists(path): return read_yaml(path) @@ -647,6 +650,7 @@ def domain_config_get(domain, key="", full=False, export=False): else: mode = "classic" + DomainConfigPanel = _get_DomainConfigPanel() config = DomainConfigPanel(domain) return config.get(key, mode) @@ -658,175 +662,189 @@ def domain_config_set( """ Apply a new domain configuration """ + from yunohost.utils.form import BaseOption + + DomainConfigPanel = _get_DomainConfigPanel() BaseOption.operation_logger = operation_logger config = DomainConfigPanel(domain) return config.set(key, value, args, args_file, operation_logger=operation_logger) -class DomainConfigPanel(ConfigPanel): - entity_type = "domain" - save_path_tpl = f"{DOMAIN_SETTINGS_DIR}/{{entity}}.yml" - save_mode = "diff" +def _get_DomainConfigPanel(): + from yunohost.utils.configpanel import ConfigPanel - # i18n: domain_config_cert_renew_help - # i18n: domain_config_default_app_help - # i18n: domain_config_xmpp_help + class DomainConfigPanel(ConfigPanel): + entity_type = "domain" + save_path_tpl = f"{DOMAIN_SETTINGS_DIR}/{{entity}}.yml" + save_mode = "diff" - def _get_raw_config(self) -> "RawConfig": - # TODO add mechanism to share some settings with other domains on the same zone - raw_config = super()._get_raw_config() + # i18n: domain_config_cert_renew_help + # i18n: domain_config_default_app_help + # i18n: domain_config_xmpp_help - any_filter = all(self.filter_key) - panel_id, section_id, option_id = self.filter_key + def _get_raw_config(self) -> "RawConfig": + # TODO add mechanism to share some settings with other domains on the same zone + raw_config = super()._get_raw_config() - raw_config["feature"]["xmpp"]["xmpp"]["default"] = ( - 1 if self.entity == _get_maindomain() else 0 - ) + any_filter = all(self.filter_key) + panel_id, section_id, option_id = self.filter_key - # Portal settings are only available on "topest" domains - if _get_parent_domain_of(self.entity, topest=True) is not None: - del raw_config["feature"]["portal"] - - # Optimize wether or not to load the DNS section, - # e.g. we don't want to trigger the whole _get_registary_config_section - # when just getting the current value from the feature section - if not any_filter or panel_id == "dns": - from yunohost.dns import _get_registrar_config_section - - raw_config["dns"]["registrar"] = _get_registrar_config_section(self.entity) - - # Cert stuff - if not any_filter or panel_id == "cert": - from yunohost.certificate import certificate_status - - status = certificate_status([self.entity], full=True)["certificates"][ - self.entity - ] - - raw_config["cert"]["cert"]["cert_summary"]["style"] = status["style"] - - # i18n: domain_config_cert_summary_expired - # i18n: domain_config_cert_summary_selfsigned - # i18n: domain_config_cert_summary_abouttoexpire - # i18n: domain_config_cert_summary_ok - # i18n: domain_config_cert_summary_letsencrypt - raw_config["cert"]["cert"]["cert_summary"]["ask"] = m18n.n( - f"domain_config_cert_summary_{status['summary']}" + raw_config["feature"]["xmpp"]["xmpp"]["default"] = ( + 1 if self.entity == _get_maindomain() else 0 ) - for option_id, status_key in [ - ("cert_validity", "validity"), - ("cert_issuer", "CA_type"), - ("acme_eligible", "ACME_eligible"), - # FIXME not sure why "summary" was injected in settings values - # ("summary", "summary") - ]: - raw_config["cert"]["cert"][option_id]["default"] = status[status_key] + # Portal settings are only available on "topest" domains + if _get_parent_domain_of(self.entity, topest=True) is not None: + del raw_config["feature"]["portal"] - # Other specific strings used in config panels - # i18n: domain_config_cert_renew_help + # Optimize wether or not to load the DNS section, + # e.g. we don't want to trigger the whole _get_registary_config_section + # when just getting the current value from the feature section + if not any_filter or panel_id == "dns": + from yunohost.dns import _get_registrar_config_section - return raw_config - - def _apply( - self, - form: "FormModel", - previous_settings: dict[str, Any], - exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, - ) -> None: - next_settings = { - k: v for k, v in form.dict().items() if previous_settings.get(k) != v - } - - if "default_app" in next_settings: - from yunohost.app import app_ssowatconf, app_map - - if "/" in app_map(raw=True).get(self.entity, {}): - raise YunohostValidationError( - "app_make_default_location_already_used", - app=next_settings["default_app"], - domain=self.entity, - other_app=app_map(raw=True)[self.entity]["/"]["id"], + raw_config["dns"]["registrar"] = _get_registrar_config_section( + self.entity ) - if next_settings.get("recovery_password", None): - domain_dyndns_set_recovery_password( - self.entity, next_settings["recovery_password"] - ) + # Cert stuff + if not any_filter or panel_id == "cert": + from yunohost.certificate import certificate_status - portal_options = [ - "default_app", - "show_other_domains_apps", - "portal_title", - "portal_logo", - "portal_theme", - "portal_user_intro", - "portal_public_intro", - ] + status = certificate_status([self.entity], full=True)["certificates"][ + self.entity + ] - if _get_parent_domain_of(self.entity, topest=True) is None and any( - option in next_settings for option in portal_options - ): - from yunohost.portal import PORTAL_SETTINGS_DIR + raw_config["cert"]["cert"]["cert_summary"]["style"] = status["style"] - # Portal options are also saved in a `domain.portal.yml` file - # that can be read by the portal API. - # FIXME remove those from the config panel saved values? + # i18n: domain_config_cert_summary_expired + # i18n: domain_config_cert_summary_selfsigned + # i18n: domain_config_cert_summary_abouttoexpire + # i18n: domain_config_cert_summary_ok + # i18n: domain_config_cert_summary_letsencrypt + raw_config["cert"]["cert"]["cert_summary"]["ask"] = m18n.n( + f"domain_config_cert_summary_{status['summary']}" + ) - portal_values = form.dict(include=set(portal_options)) - # Remove logo from values else filename will replace b64 content - portal_values.pop("portal_logo") + for option_id, status_key in [ + ("cert_validity", "validity"), + ("cert_issuer", "CA_type"), + ("acme_eligible", "ACME_eligible"), + # FIXME not sure why "summary" was injected in settings values + # ("summary", "summary") + ]: + raw_config["cert"]["cert"][option_id]["default"] = status[ + status_key + ] - if "portal_logo" in next_settings: - if previous_settings.get("portal_logo"): - try: - os.remove(previous_settings["portal_logo"]) - except FileNotFoundError: - logger.warning( - f"Coulnd't remove previous logo file, maybe the file was already deleted, path: {previous_settings['portal_logo']}" - ) - finally: - portal_values["portal_logo"] = "" + # Other specific strings used in config panels + # i18n: domain_config_cert_renew_help - if next_settings["portal_logo"]: - # Save the file content as `{mimetype}:{base64content}` in portal settings - # while keeping the file path in the domain settings - from base64 import b64encode - from magic import Magic + return raw_config - file_content = Path(next_settings["portal_logo"]).read_bytes() - mimetype = Magic(mime=True).from_buffer(file_content) - portal_values["portal_logo"] = ( - mimetype + ":" + b64encode(file_content).decode("utf-8") + def _apply( + self, + form: "FormModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ) -> None: + next_settings = { + k: v for k, v in form.dict().items() if previous_settings.get(k) != v + } + + if "default_app" in next_settings: + from yunohost.app import app_map + + if "/" in app_map(raw=True).get(self.entity, {}): + raise YunohostValidationError( + "app_make_default_location_already_used", + app=next_settings["default_app"], + domain=self.entity, + other_app=app_map(raw=True)[self.entity]["/"]["id"], ) - portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{self.entity}.json") - portal_settings: dict[str, Any] = {"apps": {}} + if next_settings.get("recovery_password", None): + domain_dyndns_set_recovery_password( + self.entity, next_settings["recovery_password"] + ) - if portal_settings_path.exists(): - portal_settings.update(read_json(str(portal_settings_path))) + portal_options = [ + "default_app", + "show_other_domains_apps", + "portal_title", + "portal_logo", + "portal_theme", + "portal_user_intro", + "portal_public_intro", + ] - # Merge settings since this config file is shared with `app_ssowatconf()` which populate the `apps` key. - portal_settings.update(portal_values) - write_to_json( - str(portal_settings_path), portal_settings, sort_keys=True, indent=4 - ) + if _get_parent_domain_of(self.entity, topest=True) is None and any( + option in next_settings for option in portal_options + ): + from yunohost.portal import PORTAL_SETTINGS_DIR - super()._apply(form, previous_settings, exclude={"recovery_password"}) + # Portal options are also saved in a `domain.portal.yml` file + # that can be read by the portal API. + # FIXME remove those from the config panel saved values? - # Reload ssowat if default app changed - if "default_app" in next_settings: - app_ssowatconf() + portal_values = form.dict(include=set(portal_options)) + # Remove logo from values else filename will replace b64 content + portal_values.pop("portal_logo") - stuff_to_regen_conf = set() - if "xmpp" in next_settings: - stuff_to_regen_conf.update({"nginx", "metronome"}) + if "portal_logo" in next_settings: + if previous_settings.get("portal_logo"): + try: + os.remove(previous_settings["portal_logo"]) + except FileNotFoundError: + logger.warning( + f"Coulnd't remove previous logo file, maybe the file was already deleted, path: {previous_settings['portal_logo']}" + ) + finally: + portal_values["portal_logo"] = "" - if "mail_in" in next_settings or "mail_out" in next_settings: - stuff_to_regen_conf.update({"nginx", "postfix", "dovecot", "rspamd"}) + if next_settings["portal_logo"]: + # Save the file content as `{mimetype}:{base64content}` in portal settings + # while keeping the file path in the domain settings + from base64 import b64encode + from magic import Magic - if stuff_to_regen_conf: - regen_conf(names=list(stuff_to_regen_conf)) + file_content = Path(next_settings["portal_logo"]).read_bytes() + mimetype = Magic(mime=True).from_buffer(file_content) + portal_values["portal_logo"] = ( + mimetype + ":" + b64encode(file_content).decode("utf-8") + ) + + portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{self.entity}.json") + portal_settings: dict[str, Any] = {"apps": {}} + + if portal_settings_path.exists(): + portal_settings.update(read_json(str(portal_settings_path))) + + # Merge settings since this config file is shared with `app_ssowatconf()` which populate the `apps` key. + portal_settings.update(portal_values) + write_to_json( + str(portal_settings_path), portal_settings, sort_keys=True, indent=4 + ) + + super()._apply(form, previous_settings, exclude={"recovery_password"}) + + # Reload ssowat if default app changed + if "default_app" in next_settings: + from yunohost.app import app_ssowatconf + + app_ssowatconf() + + stuff_to_regen_conf = set() + if "xmpp" in next_settings: + stuff_to_regen_conf.update({"nginx", "metronome"}) + + if "mail_in" in next_settings or "mail_out" in next_settings: + stuff_to_regen_conf.update({"nginx", "postfix", "dovecot", "rspamd"}) + + if stuff_to_regen_conf: + regen_conf(names=list(stuff_to_regen_conf)) + + return DomainConfigPanel def domain_action_run(domain, action, args=None):