From 274a219fdcbf3b4ec86d888d5b79e549cc654162 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 8 Jun 2018 23:55:42 +0200 Subject: [PATCH 01/13] [enh] allow hook_exec to have custom callbacks --- src/yunohost/hook.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 32570ab57..4688b25b3 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -297,7 +297,8 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, def hook_exec(path, args=None, raise_on_error=False, no_trace=False, - chdir=None, env=None, user="admin"): + chdir=None, env=None, user="admin", stdout_callback=None, + stderr_callback=None): """ Execute hook from a file with arguments @@ -361,8 +362,8 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, # Define output callbacks and call command callbacks = ( - lambda l: logger.debug(l.rstrip()), - lambda l: logger.warning(l.rstrip()), + stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()), + stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()), ) returncode = call_async_output( command, callbacks, shell=False, cwd=chdir From c0ec14b79be267e8b4dd9419e21e5d3ae4f43fa4 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 8 Jun 2018 23:56:24 +0200 Subject: [PATCH 02/13] [enh] first working prototype of config-panel show --- data/actionsmap/yunohost.yml | 14 +++++++ src/yunohost/app.py | 80 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 7db57b0e6..eca8e87f7 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -771,6 +771,20 @@ app: apps: nargs: "+" + subcategories: + + config: + subcategory_help: Applications configuration panel + actions: + + ### app_config_show_panel() + show-panel: + action_help: show config panel for the application + api: GET //config/panel + arguments: + app_id: + help: App ID + ############################# # Backup # ############################# diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 72a53da1f..137a9dd3e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -40,6 +40,7 @@ from collections import OrderedDict from moulinette import msignals, m18n, msettings from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_json from yunohost.service import service_log, _run_service_command from yunohost.utils import packages @@ -1360,6 +1361,85 @@ def app_change_label(auth, app, new_label): app_ssowatconf(auth) +def app_config_show_panel(app_id): + from yunohost.hook import hook_exec + + installed = _is_installed(app_id) + if not installed: + raise MoulinetteError(errno.ENOPKG, + m18n.n('app_not_installed', app=app_id)) + + config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') + config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config') + + if not os.path.exists(config_panel) or not os.path.exists(config_script): + return { + "config_panel": [], + } + + config_panel = read_json(config_panel) + + env = {"YNH_APP_ID": app_id} + parsed_values = {} + + # I need to parse stdout to communicate between scripts because I can't + # read the child environment :( (that would simplify things so much) + # after hours of research this is apparently quite a standard way, another + # option would be to add an explicite pipe or a named pipe for that + # a third option would be to write in a temporary file but I don't like + # that because that could expose sensitive data + def parse_stdout(line): + line = line.rstrip() + logger.info(line) + + if line.strip().startswith("YNH_CONFIG_") and "=" in line: + # XXX error handling? + # XXX this might not work for multilines stuff :( (but echo without + # formatting should do it no?) + key, value = line.strip().split("=", 1) + logger.debug("config script declared: %s -> %s", key, value) + parsed_values[key] = value + print "in parse_stdout", parsed_values + print [line] + + hook_exec(config_script, + args=[], + env=env, + user="root", + stdout_callback=parse_stdout, + ) + + # logger.debug("Env after running config script %s", env) + + logger.debug("Generating global variables:") + for tab in config_panel.get("panel", []): + tab_id = tab["id"] # this makes things easier to debug on crash + for section in tab.get("sections", []): + section_id = section["id"] + for option in section.get("options", []): + option_id = option["id"] + variable_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), variable_name) + + if variable_name in parsed_values: + # XXX we should probably uses the one of install here but it's at a POC state right now + option_type = option["type"] + if option_type == "bool": + assert parsed_values[variable_name].lower() in ("true", "false") + option["value"] = True if parsed_values[variable_name].lower() == "true" else False + elif option_type == "integer": + option["value"] = int(parsed_values[variable_name]) + elif option_type == "text": + option["value"] = parsed_values[variable_name] + else: + logger.debug("Variable '%s' is not declared by config script, using default", variable_name) + option["value"] = option["default"] + + return { + "config_panel": config_panel, + } + + def _get_app_settings(app_id): """ Get settings of an installed app From 1c16693cdcabbd48e9de5ee3f9335b08d10e9301 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:18:47 +0200 Subject: [PATCH 03/13] [mod] explicitely tells script/config that I want the show behavior --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 137a9dd3e..696b79db8 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1403,7 +1403,7 @@ def app_config_show_panel(app_id): print [line] hook_exec(config_script, - args=[], + args=["show"], env=env, user="root", stdout_callback=parse_stdout, From 359ef6d3d73fdaa2adc60f19cfddc3a06e2a5b42 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:21:09 +0200 Subject: [PATCH 04/13] [enh] handle failure on 'config/script show' --- src/yunohost/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 696b79db8..7154883eb 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1402,14 +1402,15 @@ def app_config_show_panel(app_id): print "in parse_stdout", parsed_values print [line] - hook_exec(config_script, + return_code = hook_exec(config_script, args=["show"], env=env, user="root", stdout_callback=parse_stdout, ) - # logger.debug("Env after running config script %s", env) + if return_code != 0: + raise Exception("script/config show return value code: %s (considered as an error)", return_code) logger.debug("Generating global variables:") for tab in config_panel.get("panel", []): From 7d634a3085a42b0f50b4de83cb8feb4d978a2075 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:21:22 +0200 Subject: [PATCH 05/13] [mod] s/variable_name/generated_id --- src/yunohost/app.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7154883eb..0cc966801 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1419,21 +1419,22 @@ def app_config_show_panel(app_id): section_id = section["id"] for option in section.get("options", []): option_id = option["id"] - variable_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() - logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), variable_name) + generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + option["id"] = generated_id + logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), generated_id) - if variable_name in parsed_values: + if generated_id in parsed_values: # XXX we should probably uses the one of install here but it's at a POC state right now option_type = option["type"] if option_type == "bool": - assert parsed_values[variable_name].lower() in ("true", "false") - option["value"] = True if parsed_values[variable_name].lower() == "true" else False + assert parsed_values[generated_id].lower() in ("true", "false") + option["value"] = True if parsed_values[generated_id].lower() == "true" else False elif option_type == "integer": - option["value"] = int(parsed_values[variable_name]) + option["value"] = int(parsed_values[generated_id]) elif option_type == "text": - option["value"] = parsed_values[variable_name] + option["value"] = parsed_values[generated_id] else: - logger.debug("Variable '%s' is not declared by config script, using default", variable_name) + logger.debug("Variable '%s' is not declared by config script, using default", generated_id) option["value"] = option["default"] return { From ff3913adbb7d2b05c11a1896c5aeb06839ab96da Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:21:39 +0200 Subject: [PATCH 06/13] [enh] first version of script/config apply --- data/actionsmap/yunohost.yml | 11 ++++++++ src/yunohost/app.py | 51 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index eca8e87f7..03f9a214c 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -785,6 +785,17 @@ app: app_id: help: App ID + ### app_config_apply() + apply: + action_help: apply the new configuration + api: POST //config + arguments: + app_id: + help: App ID + -a: + full: --args + help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path") + ############################# # Backup # ############################# diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0cc966801..65438dadf 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1442,6 +1442,57 @@ def app_config_show_panel(app_id): } +def app_config_apply(app_id, args): + from yunohost.hook import hook_exec + + installed = _is_installed(app_id) + if not installed: + raise MoulinetteError(errno.ENOPKG, + m18n.n('app_not_installed', app=app_id)) + + config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') + config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config') + + if not os.path.exists(config_panel) or not os.path.exists(config_script): + # XXX real exception + raise Exception("Not config-panel.json nor scripts/config") + + config_panel = read_json(config_panel) + + env = {} + args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) + + for tab in config_panel.get("panel", []): + tab_id = tab["id"] # this makes things easier to debug on crash + for section in tab.get("sections", []): + section_id = section["id"] + for option in section.get("options", []): + option_id = option["id"] + generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + + if generated_id in args: + logger.debug("include into env %s=%s", generated_id, args[generated_id]) + env[generated_id] = args[generated_id] + else: + logger.debug("no value for key id %s", generated_id) + + # for debug purpose + for key in args: + if key not in env: + logger.warning("Ignore key '%s' from arguments because it is not in the config", key) + + return_code = hook_exec(config_script, + args=["apply"], + env=env, + user="root", + ) + + if return_code != 0: + raise Exception("'script/config apply' return value code: %s (considered as an error)", return_code) + + logger.success("Config updated as expected") + + def _get_app_settings(app_id): """ Get settings of an installed app From ffbe059d9138bf89e8e5bf2516c77215a42f7494 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 10 Jun 2018 01:01:52 +0200 Subject: [PATCH 07/13] [mod] remove debug output --- src/yunohost/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 65438dadf..ca98d7a14 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1399,8 +1399,6 @@ def app_config_show_panel(app_id): key, value = line.strip().split("=", 1) logger.debug("config script declared: %s -> %s", key, value) parsed_values[key] = value - print "in parse_stdout", parsed_values - print [line] return_code = hook_exec(config_script, args=["show"], From 27dfd6af6e4e504793093c9db6d09025e36303c6 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 10 Jun 2018 20:35:27 +0200 Subject: [PATCH 08/13] [fix] missing YNH_APP_ID on scripts/config apply --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index ca98d7a14..7cbc8922b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1457,7 +1457,7 @@ def app_config_apply(app_id, args): config_panel = read_json(config_panel) - env = {} + env = {"YNH_APP_ID": app_id} args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) for tab in config_panel.get("panel", []): From 67ba031eabe70e5ce67f4961959b6008a25ecd7b Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 10 Jun 2018 20:39:20 +0200 Subject: [PATCH 09/13] [mod] copy config_panel.json on install/upgrade --- src/yunohost/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7cbc8922b..64806fbcf 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -641,6 +641,9 @@ def app_upgrade(auth, app=[], url=None, file=None): os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path)) os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): + os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "conf")): os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path)) @@ -758,6 +761,9 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False): os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): + os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "conf")): os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path)) From 6a1b8fdfa360a6bf7edf0216be4a1bb4224b4f87 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 18 Jun 2018 04:02:23 +0200 Subject: [PATCH 10/13] [mod] better API paths for configpanel --- data/actionsmap/yunohost.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 03f9a214c..d276a6cef 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -780,7 +780,7 @@ app: ### app_config_show_panel() show-panel: action_help: show config panel for the application - api: GET //config/panel + api: GET /apps//config-panel arguments: app_id: help: App ID @@ -788,7 +788,7 @@ app: ### app_config_apply() apply: action_help: apply the new configuration - api: POST //config + api: POST /apps//config arguments: app_id: help: App ID From 6fb19edbb3b1dcbf037ad58a83475713a831cc4d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 18 Jun 2018 04:03:00 +0200 Subject: [PATCH 11/13] [enh] send app name and id to config-panel-show --- src/yunohost/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 64806fbcf..32ecff82a 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1370,10 +1370,8 @@ def app_change_label(auth, app, new_label): def app_config_show_panel(app_id): from yunohost.hook import hook_exec - installed = _is_installed(app_id) - if not installed: - raise MoulinetteError(errno.ENOPKG, - m18n.n('app_not_installed', app=app_id)) + # this will take care of checking if the app is installed + app_info_dict = app_info(app_id) config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config') @@ -1442,6 +1440,8 @@ def app_config_show_panel(app_id): option["value"] = option["default"] return { + "app_id": app_id, + "app_name": app_info_dict["name"], "config_panel": config_panel, } From 8a33199d916027d6596c65c0df1a2533fcbbd179 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 04:31:13 +0200 Subject: [PATCH 12/13] [mod] add experimental warning for config panel --- locales/en.json | 1 + src/yunohost/app.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/locales/en.json b/locales/en.json index dc7a5203a..19213c372 100644 --- a/locales/en.json +++ b/locales/en.json @@ -179,6 +179,7 @@ "executing_command": "Executing command '{command:s}'...", "executing_script": "Executing script '{script:s}'...", "extracting": "Extracting...", + "experimental_feature": "Warning: this feature is experimental and not consider stable, you shouldn't be using it except if you know what you are doing.", "field_invalid": "Invalid field '{:s}'", "firewall_reload_failed": "Unable to reload the firewall", "firewall_reloaded": "The firewall has been reloaded", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 32ecff82a..2a7b6bf74 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1367,7 +1367,12 @@ def app_change_label(auth, app, new_label): app_ssowatconf(auth) +# Config panel todo list: +# * docstrings +# * merge translations on the json once the workflow is in place def app_config_show_panel(app_id): + logger.warning(m18n.n('experimental_feature')) + from yunohost.hook import hook_exec # this will take care of checking if the app is installed @@ -1447,6 +1452,8 @@ def app_config_show_panel(app_id): def app_config_apply(app_id, args): + logger.warning(m18n.n('experimental_feature')) + from yunohost.hook import hook_exec installed = _is_installed(app_id) From c14077dc3656b7e9f42cf6eb579e1daa1e0c1766 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 04:31:51 +0200 Subject: [PATCH 13/13] [fix] handle empty args case --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 2a7b6bf74..5f52a8628 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1471,7 +1471,7 @@ def app_config_apply(app_id, args): config_panel = read_json(config_panel) env = {"YNH_APP_ID": app_id} - args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) + args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} for tab in config_panel.get("panel", []): tab_id = tab["id"] # this makes things easier to debug on crash