From ff565355689db47159241521f7d750f8cb309e58 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Thu, 21 Jun 2018 12:48:52 +0200 Subject: [PATCH 01/50] [fix] Mail permission issue after restore --- data/hooks/restore/23-data_mail | 1 + 1 file changed, 1 insertion(+) diff --git a/data/hooks/restore/23-data_mail b/data/hooks/restore/23-data_mail index 99530827..81b9b923 100644 --- a/data/hooks/restore/23-data_mail +++ b/data/hooks/restore/23-data_mail @@ -1,6 +1,7 @@ backup_dir="$1/data/mail" sudo cp -a $backup_dir/. /var/mail/ || echo 'No mail found' +sudo chown -R vmail:mail /var/mail/ # Restart services to use migrated certs sudo service postfix restart From 274a219fdcbf3b4ec86d888d5b79e549cc654162 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 8 Jun 2018 23:55:42 +0200 Subject: [PATCH 02/50] [enh] allow hook_exec to have custom callbacks --- src/yunohost/hook.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 32570ab5..4688b25b 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -297,7 +297,8 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, def hook_exec(path, args=None, raise_on_error=False, no_trace=False, - chdir=None, env=None, user="admin"): + chdir=None, env=None, user="admin", stdout_callback=None, + stderr_callback=None): """ Execute hook from a file with arguments @@ -361,8 +362,8 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, # Define output callbacks and call command callbacks = ( - lambda l: logger.debug(l.rstrip()), - lambda l: logger.warning(l.rstrip()), + stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()), + stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()), ) returncode = call_async_output( command, callbacks, shell=False, cwd=chdir From c0ec14b79be267e8b4dd9419e21e5d3ae4f43fa4 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 8 Jun 2018 23:56:24 +0200 Subject: [PATCH 03/50] [enh] first working prototype of config-panel show --- data/actionsmap/yunohost.yml | 14 +++++++ src/yunohost/app.py | 80 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 7db57b0e..eca8e87f 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -771,6 +771,20 @@ app: apps: nargs: "+" + subcategories: + + config: + subcategory_help: Applications configuration panel + actions: + + ### app_config_show_panel() + show-panel: + action_help: show config panel for the application + api: GET //config/panel + arguments: + app_id: + help: App ID + ############################# # Backup # ############################# diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 72a53da1..137a9dd3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -40,6 +40,7 @@ from collections import OrderedDict from moulinette import msignals, m18n, msettings from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_json from yunohost.service import service_log, _run_service_command from yunohost.utils import packages @@ -1360,6 +1361,85 @@ def app_change_label(auth, app, new_label): app_ssowatconf(auth) +def app_config_show_panel(app_id): + from yunohost.hook import hook_exec + + installed = _is_installed(app_id) + if not installed: + raise MoulinetteError(errno.ENOPKG, + m18n.n('app_not_installed', app=app_id)) + + config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') + config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config') + + if not os.path.exists(config_panel) or not os.path.exists(config_script): + return { + "config_panel": [], + } + + config_panel = read_json(config_panel) + + env = {"YNH_APP_ID": app_id} + parsed_values = {} + + # I need to parse stdout to communicate between scripts because I can't + # read the child environment :( (that would simplify things so much) + # after hours of research this is apparently quite a standard way, another + # option would be to add an explicite pipe or a named pipe for that + # a third option would be to write in a temporary file but I don't like + # that because that could expose sensitive data + def parse_stdout(line): + line = line.rstrip() + logger.info(line) + + if line.strip().startswith("YNH_CONFIG_") and "=" in line: + # XXX error handling? + # XXX this might not work for multilines stuff :( (but echo without + # formatting should do it no?) + key, value = line.strip().split("=", 1) + logger.debug("config script declared: %s -> %s", key, value) + parsed_values[key] = value + print "in parse_stdout", parsed_values + print [line] + + hook_exec(config_script, + args=[], + env=env, + user="root", + stdout_callback=parse_stdout, + ) + + # logger.debug("Env after running config script %s", env) + + logger.debug("Generating global variables:") + for tab in config_panel.get("panel", []): + tab_id = tab["id"] # this makes things easier to debug on crash + for section in tab.get("sections", []): + section_id = section["id"] + for option in section.get("options", []): + option_id = option["id"] + variable_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), variable_name) + + if variable_name in parsed_values: + # XXX we should probably uses the one of install here but it's at a POC state right now + option_type = option["type"] + if option_type == "bool": + assert parsed_values[variable_name].lower() in ("true", "false") + option["value"] = True if parsed_values[variable_name].lower() == "true" else False + elif option_type == "integer": + option["value"] = int(parsed_values[variable_name]) + elif option_type == "text": + option["value"] = parsed_values[variable_name] + else: + logger.debug("Variable '%s' is not declared by config script, using default", variable_name) + option["value"] = option["default"] + + return { + "config_panel": config_panel, + } + + def _get_app_settings(app_id): """ Get settings of an installed app From 1c16693cdcabbd48e9de5ee3f9335b08d10e9301 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:18:47 +0200 Subject: [PATCH 04/50] [mod] explicitely tells script/config that I want the show behavior --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 137a9dd3..696b79db 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1403,7 +1403,7 @@ def app_config_show_panel(app_id): print [line] hook_exec(config_script, - args=[], + args=["show"], env=env, user="root", stdout_callback=parse_stdout, From 359ef6d3d73fdaa2adc60f19cfddc3a06e2a5b42 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:21:09 +0200 Subject: [PATCH 05/50] [enh] handle failure on 'config/script show' --- src/yunohost/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 696b79db..7154883e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1402,14 +1402,15 @@ def app_config_show_panel(app_id): print "in parse_stdout", parsed_values print [line] - hook_exec(config_script, + return_code = hook_exec(config_script, args=["show"], env=env, user="root", stdout_callback=parse_stdout, ) - # logger.debug("Env after running config script %s", env) + if return_code != 0: + raise Exception("script/config show return value code: %s (considered as an error)", return_code) logger.debug("Generating global variables:") for tab in config_panel.get("panel", []): From 7d634a3085a42b0f50b4de83cb8feb4d978a2075 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:21:22 +0200 Subject: [PATCH 06/50] [mod] s/variable_name/generated_id --- src/yunohost/app.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7154883e..0cc96680 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1419,21 +1419,22 @@ def app_config_show_panel(app_id): section_id = section["id"] for option in section.get("options", []): option_id = option["id"] - variable_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() - logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), variable_name) + generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + option["id"] = generated_id + logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), generated_id) - if variable_name in parsed_values: + if generated_id in parsed_values: # XXX we should probably uses the one of install here but it's at a POC state right now option_type = option["type"] if option_type == "bool": - assert parsed_values[variable_name].lower() in ("true", "false") - option["value"] = True if parsed_values[variable_name].lower() == "true" else False + assert parsed_values[generated_id].lower() in ("true", "false") + option["value"] = True if parsed_values[generated_id].lower() == "true" else False elif option_type == "integer": - option["value"] = int(parsed_values[variable_name]) + option["value"] = int(parsed_values[generated_id]) elif option_type == "text": - option["value"] = parsed_values[variable_name] + option["value"] = parsed_values[generated_id] else: - logger.debug("Variable '%s' is not declared by config script, using default", variable_name) + logger.debug("Variable '%s' is not declared by config script, using default", generated_id) option["value"] = option["default"] return { From ff3913adbb7d2b05c11a1896c5aeb06839ab96da Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 9 Jun 2018 23:21:39 +0200 Subject: [PATCH 07/50] [enh] first version of script/config apply --- data/actionsmap/yunohost.yml | 11 ++++++++ src/yunohost/app.py | 51 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index eca8e87f..03f9a214 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -785,6 +785,17 @@ app: app_id: help: App ID + ### app_config_apply() + apply: + action_help: apply the new configuration + api: POST //config + arguments: + app_id: + help: App ID + -a: + full: --args + help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path") + ############################# # Backup # ############################# diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0cc96680..65438dad 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1442,6 +1442,57 @@ def app_config_show_panel(app_id): } +def app_config_apply(app_id, args): + from yunohost.hook import hook_exec + + installed = _is_installed(app_id) + if not installed: + raise MoulinetteError(errno.ENOPKG, + m18n.n('app_not_installed', app=app_id)) + + config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') + config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config') + + if not os.path.exists(config_panel) or not os.path.exists(config_script): + # XXX real exception + raise Exception("Not config-panel.json nor scripts/config") + + config_panel = read_json(config_panel) + + env = {} + args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) + + for tab in config_panel.get("panel", []): + tab_id = tab["id"] # this makes things easier to debug on crash + for section in tab.get("sections", []): + section_id = section["id"] + for option in section.get("options", []): + option_id = option["id"] + generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + + if generated_id in args: + logger.debug("include into env %s=%s", generated_id, args[generated_id]) + env[generated_id] = args[generated_id] + else: + logger.debug("no value for key id %s", generated_id) + + # for debug purpose + for key in args: + if key not in env: + logger.warning("Ignore key '%s' from arguments because it is not in the config", key) + + return_code = hook_exec(config_script, + args=["apply"], + env=env, + user="root", + ) + + if return_code != 0: + raise Exception("'script/config apply' return value code: %s (considered as an error)", return_code) + + logger.success("Config updated as expected") + + def _get_app_settings(app_id): """ Get settings of an installed app From ffbe059d9138bf89e8e5bf2516c77215a42f7494 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 10 Jun 2018 01:01:52 +0200 Subject: [PATCH 08/50] [mod] remove debug output --- src/yunohost/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 65438dad..ca98d7a1 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1399,8 +1399,6 @@ def app_config_show_panel(app_id): key, value = line.strip().split("=", 1) logger.debug("config script declared: %s -> %s", key, value) parsed_values[key] = value - print "in parse_stdout", parsed_values - print [line] return_code = hook_exec(config_script, args=["show"], From 27dfd6af6e4e504793093c9db6d09025e36303c6 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 10 Jun 2018 20:35:27 +0200 Subject: [PATCH 09/50] [fix] missing YNH_APP_ID on scripts/config apply --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index ca98d7a1..7cbc8922 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1457,7 +1457,7 @@ def app_config_apply(app_id, args): config_panel = read_json(config_panel) - env = {} + env = {"YNH_APP_ID": app_id} args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) for tab in config_panel.get("panel", []): From 67ba031eabe70e5ce67f4961959b6008a25ecd7b Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 10 Jun 2018 20:39:20 +0200 Subject: [PATCH 10/50] [mod] copy config_panel.json on install/upgrade --- src/yunohost/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7cbc8922..64806fbc 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -641,6 +641,9 @@ def app_upgrade(auth, app=[], url=None, file=None): os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path)) os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): + os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "conf")): os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path)) @@ -758,6 +761,9 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False): os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): + os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "conf")): os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path)) From 6a1b8fdfa360a6bf7edf0216be4a1bb4224b4f87 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 18 Jun 2018 04:02:23 +0200 Subject: [PATCH 11/50] [mod] better API paths for configpanel --- data/actionsmap/yunohost.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 03f9a214..d276a6ce 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -780,7 +780,7 @@ app: ### app_config_show_panel() show-panel: action_help: show config panel for the application - api: GET //config/panel + api: GET /apps//config-panel arguments: app_id: help: App ID @@ -788,7 +788,7 @@ app: ### app_config_apply() apply: action_help: apply the new configuration - api: POST //config + api: POST /apps//config arguments: app_id: help: App ID From 6fb19edbb3b1dcbf037ad58a83475713a831cc4d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 18 Jun 2018 04:03:00 +0200 Subject: [PATCH 12/50] [enh] send app name and id to config-panel-show --- src/yunohost/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 64806fbc..32ecff82 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1370,10 +1370,8 @@ def app_change_label(auth, app, new_label): def app_config_show_panel(app_id): from yunohost.hook import hook_exec - installed = _is_installed(app_id) - if not installed: - raise MoulinetteError(errno.ENOPKG, - m18n.n('app_not_installed', app=app_id)) + # this will take care of checking if the app is installed + app_info_dict = app_info(app_id) config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config') @@ -1442,6 +1440,8 @@ def app_config_show_panel(app_id): option["value"] = option["default"] return { + "app_id": app_id, + "app_name": app_info_dict["name"], "config_panel": config_panel, } From 8a33199d916027d6596c65c0df1a2533fcbbd179 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 04:31:13 +0200 Subject: [PATCH 13/50] [mod] add experimental warning for config panel --- locales/en.json | 1 + src/yunohost/app.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/locales/en.json b/locales/en.json index dc7a5203..19213c37 100644 --- a/locales/en.json +++ b/locales/en.json @@ -179,6 +179,7 @@ "executing_command": "Executing command '{command:s}'...", "executing_script": "Executing script '{script:s}'...", "extracting": "Extracting...", + "experimental_feature": "Warning: this feature is experimental and not consider stable, you shouldn't be using it except if you know what you are doing.", "field_invalid": "Invalid field '{:s}'", "firewall_reload_failed": "Unable to reload the firewall", "firewall_reloaded": "The firewall has been reloaded", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 32ecff82..2a7b6bf7 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1367,7 +1367,12 @@ def app_change_label(auth, app, new_label): app_ssowatconf(auth) +# Config panel todo list: +# * docstrings +# * merge translations on the json once the workflow is in place def app_config_show_panel(app_id): + logger.warning(m18n.n('experimental_feature')) + from yunohost.hook import hook_exec # this will take care of checking if the app is installed @@ -1447,6 +1452,8 @@ def app_config_show_panel(app_id): def app_config_apply(app_id, args): + logger.warning(m18n.n('experimental_feature')) + from yunohost.hook import hook_exec installed = _is_installed(app_id) From c14077dc3656b7e9f42cf6eb579e1daa1e0c1766 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 04:31:51 +0200 Subject: [PATCH 14/50] [fix] handle empty args case --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 2a7b6bf7..5f52a862 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1471,7 +1471,7 @@ def app_config_apply(app_id, args): config_panel = read_json(config_panel) env = {"YNH_APP_ID": app_id} - args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) + args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} for tab in config_panel.get("panel", []): tab_id = tab["id"] # this makes things easier to debug on crash From eef1b6d65818b682f44e919450960187a377c288 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 30 May 2018 10:38:30 +0200 Subject: [PATCH 15/50] [enh] new command to list apps action --- data/actionsmap/yunohost.yml | 12 ++++++++++++ src/yunohost/app.py | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index d276a6ce..298332c5 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -773,6 +773,18 @@ app: subcategories: + action: + subcategory_help: Handle apps actions + actions: + + ### app_action_list() + list: + action_help: List app actions + api: GET //actions + arguments: + app_id: + help: app id + config: subcategory_help: Applications configuration panel actions: diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 5f52a862..601bc7ef 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1367,6 +1367,31 @@ def app_change_label(auth, app, new_label): app_ssowatconf(auth) +# ACTIONS todo list +# * save actions.json +# commands: +# yunohost app action list $app +# yunohost app action run $app $action -d parameters +# docstring + +def app_action_list(app_id): + installed = _is_installed(app_id) + if not installed: + raise MoulinetteError(errno.ENOPKG, + m18n.n('app_not_installed', app=app_id)) + + actions = os.path.join( APPS_SETTING_PATH, app_id, 'actions.json') + + if not os.path.exists(actions): + return { + "actions": [], + } + + return { + "actions": read_json(actions), + } + + # Config panel todo list: # * docstrings # * merge translations on the json once the workflow is in place From f35e3ef0551d0f37a26736e31cc1f6123e0f5113 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 31 May 2018 12:40:13 +0200 Subject: [PATCH 16/50] [enh] can run the action of an app --- data/actionsmap/yunohost.yml | 14 +++ src/yunohost/app.py | 199 ++++++++++++++++++++++++++++++++++- src/yunohost/hook.py | 1 + 3 files changed, 212 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 298332c5..147e08b0 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -785,6 +785,20 @@ app: app_id: help: app id + ### app_action_run() + run: + action_help: Run app action + api: PUT //actions/ + arguments: + app_id: + help: app id + action: + help: action id + -a: + full: --args + help: Serialized arguments for app script (i.e. "domain=domain.tld&path=/path") + + config: subcategory_help: Applications configuration panel actions: diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 601bc7ef..05acc40e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1392,6 +1392,51 @@ def app_action_list(app_id): } +def app_action_run(app_id, action, args=None): + from yunohost.hook import hook_exec + import tempfile + + # will raise if action doesn't exist + actions = app_action_list(app_id)["actions"] + + if action not in actions: + raise MoulinetteError(errno.EINVAL, "action '%s' not available for app '%s', available actions are: %s" % (action, app_id, ", ".join(actions.keys()))) + + action_declaration = actions[action] + + # Retrieve arguments list for install script + args_dict = {} if not args else \ + dict(urlparse.parse_qsl(args, keep_blank_values=True)) + args_odict = _parse_args_for_action(actions[action], args=args_dict) + args_list = args_odict.values() + + env_dict = _make_environment_dict(args_odict, prefix="ACTION_") + env_dict["YNH_APP_ID"] = app_id + env_dict["YNH_ACTION"] = action + + _, path = tempfile.mkstemp() + + with open(path, "w") as script: + script.write(action_declaration["command"]) + + os.chmod(path, 700) + + retcode = hook_exec( + path, + args=args_list, + env=env_dict, + chdir=action_declaration.get("cwd"), + user=action_declaration.get("user", "root"), + ) + + if retcode not in action_declaration.get("accepted_return_codes", [0]): + raise MoulinetteError(retcode, "Error while executing action '%s' of app '%s': return code %s" % (action, app_id, retcode)) + + os.remove(path) + + return logger.success("Action successed!") + + # Config panel todo list: # * docstrings # * merge translations on the json once the workflow is in place @@ -2105,7 +2150,157 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): return args_dict -def _make_environment_dict(args_dict): +def _parse_args_for_action(action, args={}, auth=None): + """Parse arguments needed for an action from the actions list + + Retrieve specified arguments for the action from the manifest, and parse + given args according to that. If some required arguments are not provided, + its values will be asked if interaction is possible. + Parsed arguments will be returned as an OrderedDict + + Keyword arguments: + action -- The action + args -- A dictionnary of arguments to parse + + """ + from yunohost.domain import (domain_list, _get_maindomain, + domain_url_available, _normalize_domain_path) + from yunohost.user import user_info + + args_dict = OrderedDict() + + if 'arguments' not in action: + logger.debug("no arguments found for '%s' in manifest", action) + return args_dict + + action_args = action['arguments'] + + for arg in action_args: + arg_name = arg['name'] + arg_type = arg.get('type', 'string') + arg_default = arg.get('default', None) + arg_choices = arg.get('choices', []) + arg_value = None + + # Transpose default value for boolean type and set it to + # false if not defined. + if arg_type == 'boolean': + arg_default = 1 if arg_default else 0 + + # Attempt to retrieve argument value + if arg_name in args: + arg_value = args[arg_name] + else: + if 'ask' in arg: + # Retrieve proper ask string + ask_string = _value_for_locale(arg['ask']) + + # Append extra strings + if arg_type == 'boolean': + ask_string += ' [0 | 1]' + elif arg_choices: + ask_string += ' [{0}]'.format(' | '.join(arg_choices)) + if arg_default is not None: + ask_string += ' (default: {0})'.format(arg_default) + + # Check for a password argument + is_password = True if arg_type == 'password' else False + + if arg_type == 'domain': + arg_default = _get_maindomain() + ask_string += ' (default: {0})'.format(arg_default) + msignals.display(m18n.n('domains_available')) + for domain in domain_list(auth)['domains']: + msignals.display("- {}".format(domain)) + + try: + input_string = msignals.prompt(ask_string, is_password) + except NotImplementedError: + input_string = None + if (input_string == '' or input_string is None) \ + and arg_default is not None: + arg_value = arg_default + else: + arg_value = input_string + elif arg_default is not None: + arg_value = arg_default + + # Validate argument value + if (arg_value is None or arg_value == '') \ + and not arg.get('optional', False): + raise MoulinetteError(errno.EINVAL, + m18n.n('app_argument_required', name=arg_name)) + elif arg_value is None: + args_dict[arg_name] = '' + continue + + # Validate argument choice + if arg_choices and arg_value not in arg_choices: + raise MoulinetteError(errno.EINVAL, + m18n.n('app_argument_choice_invalid', + name=arg_name, choices=', '.join(arg_choices))) + + # Validate argument type + if arg_type == 'domain': + if arg_value not in domain_list(auth)['domains']: + raise MoulinetteError(errno.EINVAL, + m18n.n('app_argument_invalid', + name=arg_name, error=m18n.n('domain_unknown'))) + elif arg_type == 'user': + try: + user_info(auth, arg_value) + except MoulinetteError as e: + raise MoulinetteError(errno.EINVAL, + m18n.n('app_argument_invalid', + name=arg_name, error=e.strerror)) + elif arg_type == 'app': + if not _is_installed(arg_value): + raise MoulinetteError(errno.EINVAL, + m18n.n('app_argument_invalid', + name=arg_name, error=m18n.n('app_unknown'))) + elif arg_type == 'boolean': + if isinstance(arg_value, bool): + arg_value = 1 if arg_value else 0 + else: + try: + arg_value = int(arg_value) + if arg_value not in [0, 1]: + raise ValueError() + except (TypeError, ValueError): + raise MoulinetteError(errno.EINVAL, + m18n.n('app_argument_choice_invalid', + name=arg_name, choices='0, 1')) + args_dict[arg_name] = arg_value + + # END loop over action_args... + + # If there's only one "domain" and "path", validate that domain/path + # is an available url and normalize the path. + + domain_args = [arg["name"] for arg in action_args + if arg.get("type", "string") == "domain"] + path_args = [arg["name"] for arg in action_args + if arg.get("type", "string") == "path"] + + if len(domain_args) == 1 and len(path_args) == 1: + + domain = args_dict[domain_args[0]] + path = args_dict[path_args[0]] + domain, path = _normalize_domain_path(domain, path) + + # Check the url is available + if not domain_url_available(auth, domain, path): + raise MoulinetteError(errno.EINVAL, + m18n.n('app_location_unavailable')) + + # (We save this normalized path so that the install script have a + # standard path format to deal with no matter what the user inputted) + args_dict[path_args[0]] = path + + return args_dict + + +def _make_environment_dict(args_dict, prefix="APP_ARG_"): """ Convert a dictionnary containing manifest arguments to a dictionnary of env. var. to be passed to scripts @@ -2116,7 +2311,7 @@ def _make_environment_dict(args_dict): """ env_dict = {} for arg_name, arg_value in args_dict.items(): - env_dict["YNH_APP_ARG_%s" % arg_name.upper()] = arg_value + env_dict["YNH_%s%s" % (prefix, arg_name.upper())] = arg_value return env_dict diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 4688b25b..2fb179f8 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -365,6 +365,7 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()), stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()), ) + logger.debug("About to run the command '%s'" % command) returncode = call_async_output( command, callbacks, shell=False, cwd=chdir ) From 9cefc601610a99477a156c089d6026fe068d7bbd Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 31 May 2018 12:50:03 +0200 Subject: [PATCH 17/50] [enh] store actions.json en install/upgrade --- src/yunohost/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 05acc40e..e74bb0b7 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -641,6 +641,9 @@ def app_upgrade(auth, app=[], url=None, file=None): os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path)) os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "actions.json")): + os.system('cp -R %s/actions.json %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) @@ -761,6 +764,9 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False): os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "actions.json")): + os.system('cp -R %s/actions.json %s' % (extracted_app_folder, app_setting_path)) + if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) From 2d0b5edd2336e674e8d1533b730198d83c24917d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 04:52:38 +0200 Subject: [PATCH 18/50] [mod] DRY, easier to maintain here --- src/yunohost/app.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e74bb0b7..83dd4932 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -641,14 +641,9 @@ def app_upgrade(auth, app=[], url=None, file=None): os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path)) os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path)) - if os.path.exists(os.path.join(extracted_app_folder, "actions.json")): - os.system('cp -R %s/actions.json %s' % (extracted_app_folder, app_setting_path)) - - if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): - os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) - - if os.path.exists(os.path.join(extracted_app_folder, "conf")): - os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path)) + for file_to_copy in ["actions.json", "config_panel.json", "conf"]: + if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): + os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) # So much win upgraded_apps.append(app_instance_name) @@ -764,14 +759,9 @@ def app_install(auth, app, label=None, args=None, no_remove_on_failure=False): os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path)) - if os.path.exists(os.path.join(extracted_app_folder, "actions.json")): - os.system('cp -R %s/actions.json %s' % (extracted_app_folder, app_setting_path)) - - if os.path.exists(os.path.join(extracted_app_folder, "config_panel.json")): - os.system('cp -R %s/config_panel.json %s' % (extracted_app_folder, app_setting_path)) - - if os.path.exists(os.path.join(extracted_app_folder, "conf")): - os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path)) + for file_to_copy in ["actions.json", "config_panel.json", "conf"]: + if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): + os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) # Execute the app install script install_retcode = 1 From 26f2741a7e001b472aeadea3aa117c569ea0b55b Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 04:53:52 +0200 Subject: [PATCH 19/50] [mod] mark actions commands as experimental --- src/yunohost/app.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 83dd4932..bef227db 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1363,14 +1363,12 @@ def app_change_label(auth, app, new_label): app_ssowatconf(auth) -# ACTIONS todo list -# * save actions.json -# commands: -# yunohost app action list $app -# yunohost app action run $app $action -d parameters -# docstring +# actions todo list: +# * docstring def app_action_list(app_id): + logger.warning(m18n.n('experimental_feature')) + installed = _is_installed(app_id) if not installed: raise MoulinetteError(errno.ENOPKG, @@ -1389,6 +1387,8 @@ def app_action_list(app_id): def app_action_run(app_id, action, args=None): + logger.warning(m18n.n('experimental_feature')) + from yunohost.hook import hook_exec import tempfile From 64fe357995ff5b507be0e3907a28cc6ade14217d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 04:54:56 +0200 Subject: [PATCH 20/50] [mod] compress code a bit --- src/yunohost/app.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index bef227db..d310b44d 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1374,15 +1374,10 @@ def app_action_list(app_id): raise MoulinetteError(errno.ENOPKG, m18n.n('app_not_installed', app=app_id)) - actions = os.path.join( APPS_SETTING_PATH, app_id, 'actions.json') - - if not os.path.exists(actions): - return { - "actions": [], - } + actions = os.path.join(APPS_SETTING_PATH, app_id, 'actions.json') return { - "actions": read_json(actions), + "actions": read_json(actions) if os.path.exists(actions) else [], } From 3806e2a90833782382cc065165dcc65bf6ef6bb8 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 05:27:00 +0200 Subject: [PATCH 21/50] [mod] style --- src/yunohost/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d310b44d..a6c8e209 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1396,8 +1396,7 @@ def app_action_run(app_id, action, args=None): action_declaration = actions[action] # Retrieve arguments list for install script - args_dict = {} if not args else \ - dict(urlparse.parse_qsl(args, keep_blank_values=True)) + args_dict = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} args_odict = _parse_args_for_action(actions[action], args=args_dict) args_list = args_odict.values() From f8d99e52e3daadc153d82173549ec14065488c8e Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 05:27:15 +0200 Subject: [PATCH 22/50] [mod] DRY, remove big copy/pasta --- src/yunohost/app.py | 145 ++++---------------------------------------- 1 file changed, 13 insertions(+), 132 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a6c8e209..9c5f9ac6 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2005,139 +2005,12 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): args -- A dictionnary of arguments to parse """ - from yunohost.domain import (domain_list, _get_maindomain, - domain_url_available, _normalize_domain_path) - from yunohost.user import user_info - - args_dict = OrderedDict() try: action_args = manifest['arguments'][action] except KeyError: logger.debug("no arguments found for '%s' in manifest", action) else: - for arg in action_args: - arg_name = arg['name'] - arg_type = arg.get('type', 'string') - arg_default = arg.get('default', None) - arg_choices = arg.get('choices', []) - arg_value = None - - # Transpose default value for boolean type and set it to - # false if not defined. - if arg_type == 'boolean': - arg_default = 1 if arg_default else 0 - - # Attempt to retrieve argument value - if arg_name in args: - arg_value = args[arg_name] - else: - if 'ask' in arg: - # Retrieve proper ask string - ask_string = _value_for_locale(arg['ask']) - - # Append extra strings - if arg_type == 'boolean': - ask_string += ' [0 | 1]' - elif arg_choices: - ask_string += ' [{0}]'.format(' | '.join(arg_choices)) - if arg_default is not None: - ask_string += ' (default: {0})'.format(arg_default) - - # Check for a password argument - is_password = True if arg_type == 'password' else False - - if arg_type == 'domain': - arg_default = _get_maindomain() - ask_string += ' (default: {0})'.format(arg_default) - msignals.display(m18n.n('domains_available')) - for domain in domain_list(auth)['domains']: - msignals.display("- {}".format(domain)) - - try: - input_string = msignals.prompt(ask_string, is_password) - except NotImplementedError: - input_string = None - if (input_string == '' or input_string is None) \ - and arg_default is not None: - arg_value = arg_default - else: - arg_value = input_string - elif arg_default is not None: - arg_value = arg_default - - # Validate argument value - if (arg_value is None or arg_value == '') \ - and not arg.get('optional', False): - raise MoulinetteError(errno.EINVAL, - m18n.n('app_argument_required', name=arg_name)) - elif arg_value is None: - args_dict[arg_name] = '' - continue - - # Validate argument choice - if arg_choices and arg_value not in arg_choices: - raise MoulinetteError(errno.EINVAL, - m18n.n('app_argument_choice_invalid', - name=arg_name, choices=', '.join(arg_choices))) - - # Validate argument type - if arg_type == 'domain': - if arg_value not in domain_list(auth)['domains']: - raise MoulinetteError(errno.EINVAL, - m18n.n('app_argument_invalid', - name=arg_name, error=m18n.n('domain_unknown'))) - elif arg_type == 'user': - try: - user_info(auth, arg_value) - except MoulinetteError as e: - raise MoulinetteError(errno.EINVAL, - m18n.n('app_argument_invalid', - name=arg_name, error=e.strerror)) - elif arg_type == 'app': - if not _is_installed(arg_value): - raise MoulinetteError(errno.EINVAL, - m18n.n('app_argument_invalid', - name=arg_name, error=m18n.n('app_unknown'))) - elif arg_type == 'boolean': - if isinstance(arg_value, bool): - arg_value = 1 if arg_value else 0 - else: - try: - arg_value = int(arg_value) - if arg_value not in [0, 1]: - raise ValueError() - except (TypeError, ValueError): - raise MoulinetteError(errno.EINVAL, - m18n.n('app_argument_choice_invalid', - name=arg_name, choices='0, 1')) - args_dict[arg_name] = arg_value - - # END loop over action_args... - - # If there's only one "domain" and "path", validate that domain/path - # is an available url and normalize the path. - - domain_args = [arg["name"] for arg in action_args - if arg.get("type", "string") == "domain"] - path_args = [arg["name"] for arg in action_args - if arg.get("type", "string") == "path"] - - if len(domain_args) == 1 and len(path_args) == 1: - - domain = args_dict[domain_args[0]] - path = args_dict[path_args[0]] - domain, path = _normalize_domain_path(domain, path) - - # Check the url is available - if not domain_url_available(auth, domain, path): - raise MoulinetteError(errno.EINVAL, - m18n.n('app_location_unavailable')) - - # (We save this normalized path so that the install script have a - # standard path format to deal with no matter what the user inputted) - args_dict[path_args[0]] = path - - return args_dict + return _parse_action_args_in_yunohost_format(args, action_args, auth) def _parse_args_for_action(action, args={}, auth=None): @@ -2153,10 +2026,6 @@ def _parse_args_for_action(action, args={}, auth=None): args -- A dictionnary of arguments to parse """ - from yunohost.domain import (domain_list, _get_maindomain, - domain_url_available, _normalize_domain_path) - from yunohost.user import user_info - args_dict = OrderedDict() if 'arguments' not in action: @@ -2165,6 +2034,18 @@ def _parse_args_for_action(action, args={}, auth=None): action_args = action['arguments'] + return _parse_action_args_in_yunohost_format(args, action_args, auth) + + +def _parse_action_args_in_yunohost_format(args, action_args, auth=None): + """Parse arguments store in either manifest.json or actions.json + """ + from yunohost.domain import (domain_list, _get_maindomain, + domain_url_available, _normalize_domain_path) + from yunohost.user import user_info + + args_dict = OrderedDict() + for arg in action_args: arg_name = arg['name'] arg_type = arg.get('type', 'string') From 7cbd5641bd3ed15af7fb3c4d0f035e66e86b155a Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 06:03:07 +0200 Subject: [PATCH 23/50] [fix] fix recursive super slow import in certificate.py --- src/yunohost/certificate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 6d70b9b0..bcd04664 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -40,7 +40,6 @@ from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -import yunohost.domain from yunohost.utils.network import get_public_ip from moulinette import m18n @@ -96,6 +95,8 @@ def certificate_status(auth, domain_list, full=False): full -- Display more info about the certificates """ + import yunohost.domain + # Check if old letsencrypt_ynh is installed # TODO / FIXME - Remove this in the future once the letsencrypt app is # not used anymore @@ -243,6 +244,8 @@ def _certificate_install_selfsigned(domain_list, force=False): def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=False, staging=False): + import yunohost.domain + if not os.path.exists(ACCOUNT_KEY_FILE): _generate_account_key() @@ -309,6 +312,8 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal email -- Emails root if some renewing failed """ + import yunohost.domain + # Check if old letsencrypt_ynh is installed # TODO / FIXME - Remove this in the future once the letsencrypt app is # not used anymore @@ -402,6 +407,8 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal ############################################################################### def _check_old_letsencrypt_app(): + import yunohost.domain + installedAppIds = [app["id"] for app in yunohost.app.app_list(installed=True)["apps"]] if "letsencrypt" not in installedAppIds: From 014a39fe95b06088d18d6997565fd31a1ff3f0fa Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 06:17:57 +0200 Subject: [PATCH 24/50] [ux] display human understandable choice for boolean type on installation --- src/yunohost/app.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 9c5f9ac6..30a6932e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2068,11 +2068,15 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): # Append extra strings if arg_type == 'boolean': - ask_string += ' [0 | 1]' + ask_string += ' [yes | no]' elif arg_choices: ask_string += ' [{0}]'.format(' | '.join(arg_choices)) + if arg_default is not None: - ask_string += ' (default: {0})'.format(arg_default) + if arg_type == 'boolean': + ask_string += ' (default: {0})'.format("yes" if arg_type == 1 else "no") + else: + ask_string += ' (default: {0})'.format(arg_default) # Check for a password argument is_password = True if arg_type == 'password' else False @@ -2133,14 +2137,14 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): if isinstance(arg_value, bool): arg_value = 1 if arg_value else 0 else: - try: - arg_value = int(arg_value) - if arg_value not in [0, 1]: - raise ValueError() - except (TypeError, ValueError): + if str(arg_value).lower() in ["1", "yes", "y"]: + arg_value = 1 + elif str(arg_value).lower() in ["0", "no", "n"]: + arg_value = 0 + else: raise MoulinetteError(errno.EINVAL, m18n.n('app_argument_choice_invalid', - name=arg_name, choices='0, 1')) + name=arg_name, choices='yes, no, y, n, 1, 0')) args_dict[arg_name] = arg_value # END loop over action_args... From dc0611d91b4294050347eb6b70bc3295dbefeb76 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 07:34:10 +0200 Subject: [PATCH 25/50] [mod] better routes for actions --- data/actionsmap/yunohost.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 147e08b0..53e6acae 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -780,7 +780,7 @@ app: ### app_action_list() list: action_help: List app actions - api: GET //actions + api: GET /apps//actions arguments: app_id: help: app id @@ -788,7 +788,7 @@ app: ### app_action_run() run: action_help: Run app action - api: PUT //actions/ + api: PUT /apps//actions/ arguments: app_id: help: app id @@ -798,7 +798,6 @@ app: full: --args help: Serialized arguments for app script (i.e. "domain=domain.tld&path=/path") - config: subcategory_help: Applications configuration panel actions: From b46fb467682c6b190a90bffee7c5a9ed315b8c0b Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Fri, 22 Jun 2018 07:34:24 +0200 Subject: [PATCH 26/50] [fix] forgot to commit actions format change --- src/yunohost/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 9c5f9ac6..420a5fb3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1389,6 +1389,7 @@ def app_action_run(app_id, action, args=None): # will raise if action doesn't exist actions = app_action_list(app_id)["actions"] + actions = {x["id"]: x for x in actions} if action not in actions: raise MoulinetteError(errno.EINVAL, "action '%s' not available for app '%s', available actions are: %s" % (action, app_id, ", ".join(actions.keys()))) From 79f1c1a8972ba8edaaae3c39d25d357782a71a78 Mon Sep 17 00:00:00 2001 From: Bram Date: Sat, 23 Jun 2018 00:42:07 +0200 Subject: [PATCH 27/50] [fix] referenced the wrong variable --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 30a6932e..4f4c9677 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2074,7 +2074,7 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): if arg_default is not None: if arg_type == 'boolean': - ask_string += ' (default: {0})'.format("yes" if arg_type == 1 else "no") + ask_string += ' (default: {0})'.format("yes" if arg_default == 1 else "no") else: ask_string += ' (default: {0})'.format(arg_default) From a7e85dbbba32de72b402781bb7c00bd7ee52e804 Mon Sep 17 00:00:00 2001 From: pitchum Date: Sat, 16 Jun 2018 10:15:52 +0200 Subject: [PATCH 28/50] [enh] Add MUA autoconfig. --- data/hooks/conf_regen/15-nginx | 15 +++++++++++++++ data/templates/nginx/autoconfig.tpl.xml | 19 +++++++++++++++++++ data/templates/nginx/server.tpl.conf | 4 ++++ src/yunohost/app.py | 1 + 4 files changed, 39 insertions(+) create mode 100644 data/templates/nginx/autoconfig.tpl.xml diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 03c769b6..7c114daa 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -38,12 +38,19 @@ do_pre_regen() { for domain in $domain_list; do domain_conf_dir="${nginx_conf_dir}/${domain}.d" mkdir -p "$domain_conf_dir" + mail_autoconfig_dir="${pending_dir}/var/www/.well-known/${domain}/autoconfig/mail/" + mkdir -p "$mail_autoconfig_dir" # NGINX server configuration cat server.tpl.conf \ | sed "s/{{ domain }}/${domain}/g" \ > "${nginx_conf_dir}/${domain}.conf" + cat autoconfig.tpl.xml \ + | sed "s/{{ domain }}/${domain}/g" \ + > "${mail_autoconfig_dir}/config-v1.1.xml" + + [[ $main_domain != $domain ]] \ && touch "${domain_conf_dir}/yunohost_local.conf" \ || cp yunohost_local.conf "${domain_conf_dir}/yunohost_local.conf" @@ -58,6 +65,14 @@ do_pre_regen() { || touch "${nginx_conf_dir}/${file}" done + # remove old mail-autoconfig files + autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml) + for file in $autoconfig_files; do + domain=$(basename $(readlink -f $(dirname $file)/../..)) + [[ $domain_list =~ $domain ]] \ + || (mkdir -p "$(dirname ${pending_dir}/${file})" && touch "${pending_dir}/${file}") + done + # disable default site mkdir -p "${nginx_dir}/sites-enabled" touch "${nginx_dir}/sites-enabled/default" diff --git a/data/templates/nginx/autoconfig.tpl.xml b/data/templates/nginx/autoconfig.tpl.xml new file mode 100644 index 00000000..a4264319 --- /dev/null +++ b/data/templates/nginx/autoconfig.tpl.xml @@ -0,0 +1,19 @@ + + + {{ domain }} + + {{ domain }} + 993 + SSL + password-cleartext + %EMAILLOCALPART% + + + {{ domain }} + 587 + STARTTLS + password-cleartext + %EMAILLOCALPART% + + + diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 56fc13f3..78909e3f 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -11,6 +11,10 @@ server { return 301 https://$http_host$request_uri; } + location /.well-known/autoconfig/mail { + alias /var/www/.well-known/{{ domain }}/autoconfig/mail; + } + access_log /var/log/nginx/{{ domain }}-access.log; error_log /var/log/nginx/{{ domain }}-error.log; } diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 72a53da1..f0ecbcfe 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1321,6 +1321,7 @@ def app_ssowatconf(auth): # Authorize ACME challenge url skipped_regex.append("^[^/]*/%.well%-known/acme%-challenge/.*$") + skipped_regex.append("^[^/]*/%.well%-known/autoconfig/mail/config%-v1%.1%.xml.*$") conf_dict = { 'portal_domain': main_domain, From 43b10298fccc079093f20641c51d41646bf54439 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 30 Jun 2018 11:14:54 +0200 Subject: [PATCH 29/50] [fix] timeout on get_public_ip otherwish dyndns update is stucked --- src/yunohost/utils/network.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index dec0384b..4398a80f 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -21,7 +21,9 @@ import logging import re import subprocess -from urllib import urlopen +import requests + +from requests import ConnectionError logger = logging.getLogger('yunohost.utils.network') @@ -37,8 +39,8 @@ def get_public_ip(protocol=4): raise ValueError("invalid protocol version") try: - return urlopen(url).read().strip() - except IOError: + return requests.get(url, timeout=30).content.strip() + except ConnectionError: return None From 20d6c3050397547a9394f2b4bb6ec5c9ba723fe6 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 30 Jun 2018 11:39:01 +0200 Subject: [PATCH 30/50] [fix] sometime nginx is not running --- data/hooks/conf_regen/15-nginx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 03c769b6..a3bc8a7c 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -77,7 +77,7 @@ do_post_regen() { done # Reload nginx configuration - sudo service nginx reload + pgrep nginx && sudo service nginx reload } FORCE=${2:-0} From 834088551f66764d889b7761be9eb6f7b27a1f3d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 30 Jun 2018 14:39:31 +0200 Subject: [PATCH 31/50] [fix] uses moulinette download_text utils --- src/yunohost/utils/network.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 4398a80f..871b3e6d 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -21,9 +21,7 @@ import logging import re import subprocess -import requests - -from requests import ConnectionError +from moulinette.utils.network import download_text logger = logging.getLogger('yunohost.utils.network') @@ -38,10 +36,7 @@ def get_public_ip(protocol=4): else: raise ValueError("invalid protocol version") - try: - return requests.get(url, timeout=30).content.strip() - except ConnectionError: - return None + return download_text(url, timeout=30).strip() def get_network_interfaces(): From 344ae6bbf261bb9d63c7e396c0abbd6652053953 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 1 Jul 2018 18:37:34 +0200 Subject: [PATCH 32/50] Fix nodejs removal in ynh_remove_nodejs --- data/helpers.d/nodejs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 156507c3..5111fa67 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -138,6 +138,8 @@ ynh_remove_nodejs () { then ynh_secure_remove "$n_install_dir" ynh_secure_remove "/usr/local/n" + sed --in-place "/N_PREFIX/d" /root/.bashrc + rm -f /etc/cron.daily/node_update fi } From 36c8d1a787b6d734e2d32e390dc8345434d9e4ce Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 2 Jul 2018 14:32:41 +0200 Subject: [PATCH 33/50] e.strerror don't always exists, use str(e) instead --- src/yunohost/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index ad8cfd84..05504a75 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -329,7 +329,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False): ssowat_conf = json.loads(str(json_conf.read())) except ValueError as e: raise MoulinetteError(errno.EINVAL, - m18n.n('ssowat_persistent_conf_read_error', error=e.strerror)) + m18n.n('ssowat_persistent_conf_read_error', error=str(e))) except IOError: ssowat_conf = {} @@ -343,7 +343,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False): json.dump(ssowat_conf, f, sort_keys=True, indent=4) except IOError as e: raise MoulinetteError(errno.EPERM, - m18n.n('ssowat_persistent_conf_write_error', error=e.strerror)) + m18n.n('ssowat_persistent_conf_write_error', error=str(e))) os.system('chmod 644 /etc/ssowat/conf.json.persistent') From 51175db814cc7ecd23c12f9882a6de5f2021e5ca Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 2 Jul 2018 17:04:41 +0200 Subject: [PATCH 34/50] [mod] information needed for the admin UI --- src/yunohost/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 420a5fb3..637db174 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1369,14 +1369,14 @@ def app_change_label(auth, app, new_label): def app_action_list(app_id): logger.warning(m18n.n('experimental_feature')) - installed = _is_installed(app_id) - if not installed: - raise MoulinetteError(errno.ENOPKG, - m18n.n('app_not_installed', app=app_id)) + # this will take care of checking if the app is installed + app_info_dict = app_info(app_id) actions = os.path.join(APPS_SETTING_PATH, app_id, 'actions.json') return { + "app_id": app_id, + "app_name": app_info_dict["name"], "actions": read_json(actions) if os.path.exists(actions) else [], } From 0ab51209fe9b387c369219578bac1e484b0a1288 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 3 Jul 2018 20:38:23 +0200 Subject: [PATCH 35/50] [enh] list available users on app installation user argument --- locales/en.json | 1 + src/yunohost/app.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 19213c37..45b3cdb1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -404,6 +404,7 @@ "user_unknown": "Unknown user: {user:s}", "user_update_failed": "Unable to update user", "user_updated": "The user has been updated", + "users_available": "Available users:", "yunohost_already_installed": "YunoHost is already installed", "yunohost_ca_creation_failed": "Unable to create certificate authority", "yunohost_ca_creation_success": "The local certification authority has been created.", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 420a5fb3..a3f593d3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2043,7 +2043,7 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): """ from yunohost.domain import (domain_list, _get_maindomain, domain_url_available, _normalize_domain_path) - from yunohost.user import user_info + from yunohost.user import user_info, user_list args_dict = OrderedDict() @@ -2085,6 +2085,11 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): for domain in domain_list(auth)['domains']: msignals.display("- {}".format(domain)) + if arg_type == 'user': + msignals.display(m18n.n('users_available')) + for user in user_list(auth)['users'].keys(): + msignals.display("- {}".format(user)) + try: input_string = msignals.prompt(ask_string, is_password) except NotImplementedError: From 3bcbe1941aa660c628359330aa9bfe286e38a3d8 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 3 Jul 2018 20:43:18 +0200 Subject: [PATCH 36/50] [mod] remove useless imports --- src/yunohost/user.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index bed5fb8c..bbcecc8d 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -36,7 +36,6 @@ import subprocess from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_file from yunohost.service import service_status logger = getActionLogger('yunohost.user') @@ -112,7 +111,6 @@ def user_create(auth, username, firstname, lastname, mail, password, mailbox_quota -- Mailbox size quota """ - import pwd from yunohost.domain import domain_list, _get_maindomain from yunohost.hook import hook_callback from yunohost.app import app_ssowatconf From 14387c43ebd24737065514f187275c53d794291a Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 9 Jul 2018 14:08:28 +0200 Subject: [PATCH 37/50] [mod] uses app_list installed option instead --- src/yunohost/app.py | 51 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 420a5fb3..68615118 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1284,7 +1284,7 @@ def app_ssowatconf(auth): redirected_urls = {} try: - apps_list = app_list()['apps'] + apps_list = app_list(installed=True)['apps'] except: apps_list = [] @@ -1293,31 +1293,30 @@ def app_ssowatconf(auth): return s.split(',') if s else [] for app in apps_list: - if _is_installed(app['id']): - with open(APPS_SETTING_PATH + app['id'] + '/settings.yml') as f: - app_settings = yaml.load(f) - for item in _get_setting(app_settings, 'skipped_uris'): - if item[-1:] == '/': - item = item[:-1] - skipped_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item) - for item in _get_setting(app_settings, 'skipped_regex'): - skipped_regex.append(item) - for item in _get_setting(app_settings, 'unprotected_uris'): - if item[-1:] == '/': - item = item[:-1] - unprotected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item) - for item in _get_setting(app_settings, 'unprotected_regex'): - unprotected_regex.append(item) - for item in _get_setting(app_settings, 'protected_uris'): - if item[-1:] == '/': - item = item[:-1] - protected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item) - for item in _get_setting(app_settings, 'protected_regex'): - protected_regex.append(item) - if 'redirected_urls' in app_settings: - redirected_urls.update(app_settings['redirected_urls']) - if 'redirected_regex' in app_settings: - redirected_regex.update(app_settings['redirected_regex']) + with open(APPS_SETTING_PATH + app['id'] + '/settings.yml') as f: + app_settings = yaml.load(f) + for item in _get_setting(app_settings, 'skipped_uris'): + if item[-1:] == '/': + item = item[:-1] + skipped_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item) + for item in _get_setting(app_settings, 'skipped_regex'): + skipped_regex.append(item) + for item in _get_setting(app_settings, 'unprotected_uris'): + if item[-1:] == '/': + item = item[:-1] + unprotected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item) + for item in _get_setting(app_settings, 'unprotected_regex'): + unprotected_regex.append(item) + for item in _get_setting(app_settings, 'protected_uris'): + if item[-1:] == '/': + item = item[:-1] + protected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item) + for item in _get_setting(app_settings, 'protected_regex'): + protected_regex.append(item) + if 'redirected_urls' in app_settings: + redirected_urls.update(app_settings['redirected_urls']) + if 'redirected_regex' in app_settings: + redirected_regex.update(app_settings['redirected_regex']) for domain in domains: skipped_urls.extend([domain + '/yunohost/admin', domain + '/yunohost/api']) From d27cce4af6f3b3267e4db74a6ec83f2b498beedb Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 9 Jul 2018 18:04:22 +0200 Subject: [PATCH 38/50] [mod] put code closer to usage --- src/yunohost/app.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 68615118..a9091577 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1270,10 +1270,6 @@ def app_ssowatconf(auth): main_domain = _get_maindomain() domains = domain_list(auth)['domains'] - users = {} - for username in user_list(auth)['users'].keys(): - users[username] = app_map(user=username) - skipped_urls = [] skipped_regex = [] unprotected_urls = [] @@ -1342,7 +1338,8 @@ def app_ssowatconf(auth): 'protected_regex': protected_regex, 'redirected_urls': redirected_urls, 'redirected_regex': redirected_regex, - 'users': users, + 'users': {username: app_map(user=username) + for username in user_list(auth)['users'].keys()}, } with open('/etc/ssowat/conf.json', 'w+') as f: From 2bf327970b9f9270d6d5453a68b31fa835d22392 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 9 Jul 2018 18:30:45 +0200 Subject: [PATCH 39/50] [enh] allow an application to optout of sso --- src/yunohost/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a9091577..3c654cbe 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -399,6 +399,8 @@ def app_map(app=None, raw=False, user=None): continue if 'domain' not in app_settings: continue + if 'no_sso' in app_settings: # I don't think we need to check for the value here + continue if user is not None: if ('mode' not in app_settings or ('mode' in app_settings @@ -1291,6 +1293,10 @@ def app_ssowatconf(auth): for app in apps_list: with open(APPS_SETTING_PATH + app['id'] + '/settings.yml') as f: app_settings = yaml.load(f) + + if 'no_sso' in app_settings: + continue + for item in _get_setting(app_settings, 'skipped_uris'): if item[-1:] == '/': item = item[:-1] From 99b395d82327761b0486ed5b9e2515e096aa1a8c Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 3 Jul 2018 20:43:18 +0200 Subject: [PATCH 40/50] [mod] remove useless imports --- src/yunohost/user.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index bed5fb8c..bbcecc8d 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -36,7 +36,6 @@ import subprocess from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_file from yunohost.service import service_status logger = getActionLogger('yunohost.user') @@ -112,7 +111,6 @@ def user_create(auth, username, firstname, lastname, mail, password, mailbox_quota -- Mailbox size quota """ - import pwd from yunohost.domain import domain_list, _get_maindomain from yunohost.hook import hook_callback from yunohost.app import app_ssowatconf From fbaac17870256b13c1e3304e4bbbb730c764bd3f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 19 Jul 2018 01:03:15 +0200 Subject: [PATCH 41/50] [enh] default action cwd to the app yunohost app settings folder --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index cd7a5ba7..2287723b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1416,7 +1416,7 @@ def app_action_run(app_id, action, args=None): path, args=args_list, env=env_dict, - chdir=action_declaration.get("cwd"), + chdir=action_declaration.get("cwd", "/etc/yunohost/apps/" + app_id), user=action_declaration.get("user", "root"), ) From 3facf89c7ed320a3d6ec6afb6e4aca46b4c89c08 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 19 Jul 2018 01:13:23 +0200 Subject: [PATCH 42/50] [enh] allow to uses in action cwd --- src/yunohost/app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 2287723b..7961591e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1412,11 +1412,16 @@ def app_action_run(app_id, action, args=None): os.chmod(path, 700) + if action_declaration.get("cwd"): + cwd = action_declaration["cwd"].replace("$app_id", app_id) + else: + cwd = "/etc/yunohost/apps/" + app_id + retcode = hook_exec( path, args=args_list, env=env_dict, - chdir=action_declaration.get("cwd", "/etc/yunohost/apps/" + app_id), + chdir=cwd, user=action_declaration.get("user", "root"), ) From 314fe830025035a777f36f76e3de7dd910028793 Mon Sep 17 00:00:00 2001 From: Josue-T Date: Thu, 19 Jul 2018 21:17:54 +0200 Subject: [PATCH 43/50] Fix container detection Since LXC 3.0 it's not possible to detect if we are in a container by the init process ID --- src/yunohost/tools.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 05504a75..c1b76a74 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -234,14 +234,14 @@ def _is_inside_container(): Returns True or False """ - # See https://stackoverflow.com/a/37016302 - p = subprocess.Popen("sudo cat /proc/1/sched".split(), + # See https://www.2daygeek.com/check-linux-system-physical-virtual-machine-virtualization-technology/ + p = subprocess.Popen("sudo systemd-detect-virt".split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, _ = p.communicate() - - return out.split()[1] != "(1," + container = ['lxc','lxd','docker'] + return out.split()[0] in container def tools_postinstall(domain, password, ignore_dyndns=False): From f528893b4d1a7b5edd70d215d35ebc8fa1f488ef Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 22 Jul 2018 11:24:32 +0200 Subject: [PATCH 44/50] [mod] propagate the no_checks logic to acme-tiny code --- src/yunohost/certificate.py | 7 ++++--- src/yunohost/vendor/acme_tiny/acme_tiny.py | 23 +++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index bcd04664..930bc029 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -289,7 +289,7 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F _check_domain_is_ready_for_ACME(domain) _configure_for_acme_challenge(auth, domain) - _fetch_and_enable_new_certificate(domain, staging) + _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks) _install_cron() logger.success( @@ -383,7 +383,7 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal if not no_checks: _check_domain_is_ready_for_ACME(domain) - _fetch_and_enable_new_certificate(domain, staging) + _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks) logger.success( m18n.n("certmanager_cert_renew_success", domain=domain)) @@ -521,7 +521,7 @@ def _check_acme_challenge_configuration(domain): return True -def _fetch_and_enable_new_certificate(domain, staging=False): +def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False): # Make sure tmp folder exists logger.debug("Making sure tmp folders exists...") @@ -562,6 +562,7 @@ def _fetch_and_enable_new_certificate(domain, staging=False): domain_csr_file, WEBROOT_FOLDER, log=logger, + no_checks=no_checks, CA=certification_authority) except ValueError as e: if "urn:acme:error:rateLimited" in str(e): diff --git a/src/yunohost/vendor/acme_tiny/acme_tiny.py b/src/yunohost/vendor/acme_tiny/acme_tiny.py index fa1ee4dc..f36aef87 100644 --- a/src/yunohost/vendor/acme_tiny/acme_tiny.py +++ b/src/yunohost/vendor/acme_tiny/acme_tiny.py @@ -12,7 +12,7 @@ LOGGER = logging.getLogger(__name__) LOGGER.addHandler(logging.StreamHandler()) LOGGER.setLevel(logging.INFO) -def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA): +def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, no_checks=False): # helper function base64 encode for jose spec def _b64(b): return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "") @@ -111,16 +111,17 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA): with open(wellknown_path, "w") as wellknown_file: wellknown_file.write(keyauthorization) - # check that the file is in place - wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token) - try: - resp = urlopen(wellknown_url) - resp_data = resp.read().decode('utf8').strip() - assert resp_data == keyauthorization - except (IOError, AssertionError): - os.remove(wellknown_path) - raise ValueError("Wrote file to {0}, but couldn't download {1}".format( - wellknown_path, wellknown_url)) + if not no_checks: # sometime the local g + # check that the file is in place + wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token) + try: + resp = urlopen(wellknown_url) + resp_data = resp.read().decode('utf8').strip() + assert resp_data == keyauthorization + except (IOError, AssertionError): + os.remove(wellknown_path) + raise ValueError("Wrote file to {0}, but couldn't download {1}".format( + wellknown_path, wellknown_url)) # notify challenge are met code, result = _send_signed_request(challenge['uri'], { From 886afd4d69b7d1de847057cfdce96eecfb454dc7 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 22 Jul 2018 13:38:43 +0200 Subject: [PATCH 45/50] [enh] after postinstall tell admin to had its first user --- locales/en.json | 1 + src/yunohost/tools.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/locales/en.json b/locales/en.json index 19213c37..6b1de237 100644 --- a/locales/en.json +++ b/locales/en.json @@ -304,6 +304,7 @@ "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections", "port_available": "Port {port:d} is available", "port_unavailable": "Port {port:d} is not available", + "recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create' or the admin interface.", "restore_action_required": "You must specify something to restore", "restore_already_installed_app": "An app is already installed with the id '{app:s}'", "restore_app_failed": "Unable to restore the app '{app:s}'", diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 05504a75..b7fbf5d7 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -408,6 +408,8 @@ def tools_postinstall(domain, password, ignore_dyndns=False): service_regen_conf(force=True) logger.success(m18n.n('yunohost_configured')) + logger.warning(m18n.n('recommend_to_add_first_user')) + def tools_update(ignore_apps=False, ignore_packages=False): """ From 0b86ae0467da773f4b224b9f46075ab34a8f31b3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 25 Jul 2018 16:55:41 +0200 Subject: [PATCH 46/50] Remove remaining mention of 'rmilter' in domain_add --- src/yunohost/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 54540859..913b7868 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -110,7 +110,7 @@ def domain_add(auth, domain, dyndns=False): # Don't regen these conf if we're still in postinstall if os.path.exists('/etc/yunohost/installed'): - service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'rmilter']) + service_regen_conf(names=['nginx', 'metronome', 'dnsmasq']) app_ssowatconf(auth) except: From 581eb066b970e408f14d1e8cb081d6078c8ba093 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 25 Jul 2018 18:38:57 +0000 Subject: [PATCH 47/50] [fix] Microdecision : dyndns update was broken if no IPv6 on the server --- src/yunohost/utils/network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 871b3e6d..a9602ff5 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -36,7 +36,11 @@ def get_public_ip(protocol=4): else: raise ValueError("invalid protocol version") - return download_text(url, timeout=30).strip() + try: + return download_text(url, timeout=30).strip() + except Exception as e: + logger.debug("Could not get public IPv%s : %s" % (str(protocol), str(e))) + return None def get_network_interfaces(): From 1de1b43e2f61023bab5551dbf55c6449bdd47397 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 25 Jul 2018 19:14:51 +0000 Subject: [PATCH 48/50] Avoid breaking the regen-conf if there's no .well-known mail autoconfig.xml to list --- data/hooks/conf_regen/15-nginx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 0fb2e6a2..1aafcbfa 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -66,7 +66,7 @@ do_pre_regen() { done # remove old mail-autoconfig files - autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml) + autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml 2>/dev/null || true) for file in $autoconfig_files; do domain=$(basename $(readlink -f $(dirname $file)/../..)) [[ $domain_list =~ $domain ]] \ From 4f73f1aed2a7eeddc59192fdc9078c8105b5c323 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 28 Jul 2018 22:05:29 +0200 Subject: [PATCH 49/50] [fix] avoid returning None when an empty dict is expected --- src/yunohost/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 5d11b75c..5c377059 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2018,6 +2018,7 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): action_args = manifest['arguments'][action] except KeyError: logger.debug("no arguments found for '%s' in manifest", action) + return OrderedDict() else: return _parse_action_args_in_yunohost_format(args, action_args, auth) From df2033227bf3fcd2a5d23160ae85c5dd0da6c1db Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 28 Jul 2018 22:06:52 +0200 Subject: [PATCH 50/50] [mod] refactor, don't use try/except when not needed --- src/yunohost/app.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 5c377059..cc37051e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2014,13 +2014,12 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): args -- A dictionnary of arguments to parse """ - try: - action_args = manifest['arguments'][action] - except KeyError: + if action not in manifest['arguments']: logger.debug("no arguments found for '%s' in manifest", action) return OrderedDict() - else: - return _parse_action_args_in_yunohost_format(args, action_args, auth) + + action_args = manifest['arguments'][action] + return _parse_action_args_in_yunohost_format(args, action_args, auth) def _parse_args_for_action(action, args={}, auth=None):