Merge branch 'stretch-unstable' into journals

data/actionsmap/yunohost.yml

@@ -771,6 +771,56 @@ app:
                 apps:
                     nargs: "+"
 
+    subcategories:
+
+      action:
+          subcategory_help: Handle apps actions
+          actions:
+
+              ### app_action_list()
+              list:
+                  action_help: List app actions
+                  api: GET /apps/<app_id>/actions
+                  arguments:
+                    app_id:
+                        help: app id
+
+              ### app_action_run()
+              run:
+                  action_help: Run app action
+                  api: PUT /apps/<app_id>/actions/<action>
+                  arguments:
+                    app_id:
+                        help: app id
+                    action:
+                        help: action id
+                    -a:
+                        full: --args
+                        help: Serialized arguments for app script (i.e. "domain=domain.tld&path=/path")
+
+      config:
+          subcategory_help: Applications configuration panel
+          actions:
+
+              ### app_config_show_panel()
+              show-panel:
+                  action_help: show config panel for the application
+                  api: GET /apps/<app_id>/config-panel
+                  arguments:
+                      app_id:
+                          help: App ID
+
+              ### app_config_apply()
+              apply:
+                  action_help: apply the new configuration
+                  api: POST /apps/<app_id>/config
+                  arguments:
+                      app_id:
+                          help: App ID
+                      -a:
+                          full: --args
+                          help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path")
+
 #############################
 #          Backup           #
 #############################

data/helpers.d/nodejs

@@ -138,6 +138,8 @@ ynh_remove_nodejs () {
 	then
 		ynh_secure_remove "$n_install_dir"
 		ynh_secure_remove "/usr/local/n"
+		sed --in-place "/N_PREFIX/d" /root/.bashrc
+		rm -f /etc/cron.daily/node_update
 	fi
 }
 

data/hooks/conf_regen/15-nginx

@@ -38,12 +38,19 @@ do_pre_regen() {
   for domain in $domain_list; do
     domain_conf_dir="${nginx_conf_dir}/${domain}.d"
     mkdir -p "$domain_conf_dir"
+    mail_autoconfig_dir="${pending_dir}/var/www/.well-known/${domain}/autoconfig/mail/"
+    mkdir -p "$mail_autoconfig_dir"
 
     # NGINX server configuration
     cat server.tpl.conf \
       | sed "s/{{ domain }}/${domain}/g" \
       > "${nginx_conf_dir}/${domain}.conf"
 
+    cat autoconfig.tpl.xml \
+      | sed "s/{{ domain }}/${domain}/g" \
+      > "${mail_autoconfig_dir}/config-v1.1.xml"
+
+
     [[ $main_domain != $domain ]] \
       && touch "${domain_conf_dir}/yunohost_local.conf" \
       || cp yunohost_local.conf "${domain_conf_dir}/yunohost_local.conf"
@@ -58,6 +65,14 @@ do_pre_regen() {
       || touch "${nginx_conf_dir}/${file}"
   done
 
+  # remove old mail-autoconfig files
+  autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml 2>/dev/null || true)
+  for file in $autoconfig_files; do
+    domain=$(basename $(readlink -f $(dirname $file)/../..))
+    [[ $domain_list =~ $domain ]] \
+      || (mkdir -p "$(dirname ${pending_dir}/${file})" && touch "${pending_dir}/${file}")
+  done
+
   # disable default site
   mkdir -p "${nginx_dir}/sites-enabled"
   touch "${nginx_dir}/sites-enabled/default"
@@ -77,7 +92,7 @@ do_post_regen() {
   done
 
   # Reload nginx configuration
-  sudo service nginx reload
+  pgrep nginx && sudo service nginx reload
 }
 
 FORCE=${2:-0}

data/hooks/restore/23-data_mail

@@ -1,6 +1,7 @@
 backup_dir="$1/data/mail"
 
 sudo cp -a $backup_dir/. /var/mail/ || echo 'No mail found'
+sudo chown -R vmail:mail /var/mail/
 
 # Restart services to use migrated certs
 sudo service postfix restart

data/templates/nginx/autoconfig.tpl.xml

@@ -0,0 +1,19 @@
+<clientConfig version="1.1">
+  <emailProvider id="{{ domain }}">
+    <domain>{{ domain }}</domain>
+    <incomingServer type="imap">
+      <hostname>{{ domain }}</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILLOCALPART%</username>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname>{{ domain }}</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILLOCALPART%</username>
+    </outgoingServer>
+  </emailProvider>
+</clientConfig>

data/templates/nginx/server.tpl.conf

@@ -11,6 +11,10 @@ server {
         return 301 https://$http_host$request_uri;
     }
 
+    location /.well-known/autoconfig/mail {
+        alias /var/www/.well-known/{{ domain }}/autoconfig/mail;
+    }
+
     access_log /var/log/nginx/{{ domain }}-access.log;
     error_log /var/log/nginx/{{ domain }}-error.log;
 }

locales/en.json

@@ -179,6 +179,7 @@
     "executing_command": "Executing command '{command:s}'...",
     "executing_script": "Executing script '{script:s}'...",
     "extracting": "Extracting...",
+    "experimental_feature": "Warning: this feature is experimental and not consider stable, you shouldn't be using it except if you know what you are doing.",
     "field_invalid": "Invalid field '{:s}'",
     "firewall_reload_failed": "Unable to reload the firewall",
     "firewall_reloaded": "The firewall has been reloaded",
@@ -346,6 +347,7 @@
     "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections",
     "port_available": "Port {port:d} is available",
     "port_unavailable": "Port {port:d} is not available",
+    "recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create' or the admin interface.",
     "restore_action_required": "You must specify something to restore",
     "restore_already_installed_app": "An app is already installed with the id '{app:s}'",
     "restore_app_failed": "Unable to restore the app '{app:s}'",
@@ -446,6 +448,7 @@
     "user_unknown": "Unknown user: {user:s}",
     "user_update_failed": "Unable to update user",
     "user_updated": "The user has been updated",
+    "users_available": "Available users:",
     "yunohost_already_installed": "YunoHost is already installed",
     "yunohost_ca_creation_failed": "Unable to create certificate authority",
     "yunohost_ca_creation_success": "The local certification authority has been created.",

src/yunohost/app.py

@@ -40,6 +40,7 @@ from collections import OrderedDict
 from moulinette import msignals, m18n, msettings
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_json
 
 from yunohost.service import service_log, _run_service_command
 from yunohost.utils import packages
@@ -405,6 +406,8 @@ def app_map(app=None, raw=False, user=None):
             continue
         if 'domain' not in app_settings:
             continue
+        if 'no_sso' in app_settings:  # I don't think we need to check for the value here
+            continue
         if user is not None:
             if ('mode' not in app_settings
                 or ('mode' in app_settings
@@ -659,8 +662,9 @@ def app_upgrade(auth, app=[], url=None, file=None):
             os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path))
             os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path))
 
-            if os.path.exists(os.path.join(extracted_app_folder, "conf")):
+            for file_to_copy in ["actions.json", "config_panel.json", "conf"]:
-                os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path))
+                if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)):
+                    os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path))
 
             # So much win
             upgraded_apps.append(app_instance_name)
@@ -785,8 +789,9 @@ def app_install(uo, auth, app, label=None, args=None, no_remove_on_failure=False
     os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path))
     os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path))
 
-    if os.path.exists(os.path.join(extracted_app_folder, "conf")):
+    for file_to_copy in ["actions.json", "config_panel.json", "conf"]:
-        os.system('cp -R %s/conf %s' % (extracted_app_folder, app_setting_path))
+        if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)):
+            os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path))
 
     # Execute the app install script
     install_retcode = 1
@@ -1344,10 +1349,6 @@ def app_ssowatconf(auth):
     main_domain = _get_maindomain()
     domains = domain_list(auth)['domains']
 
-    users = {}
-    for username in user_list(auth)['users'].keys():
-        users[username] = app_map(user=username)
-
     skipped_urls = []
     skipped_regex = []
     unprotected_urls = []
@@ -1358,7 +1359,7 @@ def app_ssowatconf(auth):
     redirected_urls = {}
 
     try:
-        apps_list = app_list()['apps']
+        apps_list = app_list(installed=True)['apps']
     except:
         apps_list = []
 
@@ -1367,37 +1368,41 @@ def app_ssowatconf(auth):
         return s.split(',') if s else []
 
     for app in apps_list:
-        if _is_installed(app['id']):
+        with open(APPS_SETTING_PATH + app['id'] + '/settings.yml') as f:
-            with open(APPS_SETTING_PATH + app['id'] + '/settings.yml') as f:
+            app_settings = yaml.load(f)
-                app_settings = yaml.load(f)
+
-                for item in _get_setting(app_settings, 'skipped_uris'):
+            if 'no_sso' in app_settings:
-                    if item[-1:] == '/':
+                continue
-                        item = item[:-1]
+
-                    skipped_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item)
+            for item in _get_setting(app_settings, 'skipped_uris'):
-                for item in _get_setting(app_settings, 'skipped_regex'):
+                if item[-1:] == '/':
-                    skipped_regex.append(item)
+                    item = item[:-1]
-                for item in _get_setting(app_settings, 'unprotected_uris'):
+                skipped_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item)
-                    if item[-1:] == '/':
+            for item in _get_setting(app_settings, 'skipped_regex'):
-                        item = item[:-1]
+                skipped_regex.append(item)
-                    unprotected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item)
+            for item in _get_setting(app_settings, 'unprotected_uris'):
-                for item in _get_setting(app_settings, 'unprotected_regex'):
+                if item[-1:] == '/':
-                    unprotected_regex.append(item)
+                    item = item[:-1]
-                for item in _get_setting(app_settings, 'protected_uris'):
+                unprotected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item)
-                    if item[-1:] == '/':
+            for item in _get_setting(app_settings, 'unprotected_regex'):
-                        item = item[:-1]
+                unprotected_regex.append(item)
-                    protected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item)
+            for item in _get_setting(app_settings, 'protected_uris'):
-                for item in _get_setting(app_settings, 'protected_regex'):
+                if item[-1:] == '/':
-                    protected_regex.append(item)
+                    item = item[:-1]
-                if 'redirected_urls' in app_settings:
+                protected_urls.append(app_settings['domain'] + app_settings['path'].rstrip('/') + item)
-                    redirected_urls.update(app_settings['redirected_urls'])
+            for item in _get_setting(app_settings, 'protected_regex'):
-                if 'redirected_regex' in app_settings:
+                protected_regex.append(item)
-                    redirected_regex.update(app_settings['redirected_regex'])
+            if 'redirected_urls' in app_settings:
+                redirected_urls.update(app_settings['redirected_urls'])
+            if 'redirected_regex' in app_settings:
+                redirected_regex.update(app_settings['redirected_regex'])
 
     for domain in domains:
         skipped_urls.extend([domain + '/yunohost/admin', domain + '/yunohost/api'])
 
     # Authorize ACME challenge url
     skipped_regex.append("^[^/]*/%.well%-known/acme%-challenge/.*$")
+    skipped_regex.append("^[^/]*/%.well%-known/autoconfig/mail/config%-v1%.1%.xml.*$")
 
     conf_dict = {
         'portal_domain': main_domain,
@@ -1417,7 +1422,8 @@ def app_ssowatconf(auth):
         'protected_regex': protected_regex,
         'redirected_urls': redirected_urls,
         'redirected_regex': redirected_regex,
-        'users': users,
+        'users': {username: app_map(user=username)
+                  for username in user_list(auth)['users'].keys()},
     }
 
     with open('/etc/ssowat/conf.json', 'w+') as f:
@@ -1437,6 +1443,213 @@ def app_change_label(auth, app, new_label):
     app_ssowatconf(auth)
 
 
+# actions todo list:
+# * docstring
+
+def app_action_list(app_id):
+    logger.warning(m18n.n('experimental_feature'))
+
+    # this will take care of checking if the app is installed
+    app_info_dict = app_info(app_id)
+
+    actions = os.path.join(APPS_SETTING_PATH, app_id, 'actions.json')
+
+    return {
+        "app_id": app_id,
+        "app_name": app_info_dict["name"],
+        "actions": read_json(actions) if os.path.exists(actions) else [],
+    }
+
+
+def app_action_run(app_id, action, args=None):
+    logger.warning(m18n.n('experimental_feature'))
+
+    from yunohost.hook import hook_exec
+    import tempfile
+
+    # will raise if action doesn't exist
+    actions = app_action_list(app_id)["actions"]
+    actions = {x["id"]: x for x in actions}
+
+    if action not in actions:
+        raise MoulinetteError(errno.EINVAL, "action '%s' not available for app '%s', available actions are: %s" % (action, app_id, ", ".join(actions.keys())))
+
+    action_declaration = actions[action]
+
+    # Retrieve arguments list for install script
+    args_dict = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {}
+    args_odict = _parse_args_for_action(actions[action], args=args_dict)
+    args_list = args_odict.values()
+
+    env_dict = _make_environment_dict(args_odict, prefix="ACTION_")
+    env_dict["YNH_APP_ID"] = app_id
+    env_dict["YNH_ACTION"] = action
+
+    _, path = tempfile.mkstemp()
+
+    with open(path, "w") as script:
+        script.write(action_declaration["command"])
+
+    os.chmod(path, 700)
+
+    if action_declaration.get("cwd"):
+        cwd = action_declaration["cwd"].replace("$app_id", app_id)
+    else:
+        cwd = "/etc/yunohost/apps/" + app_id
+
+    retcode = hook_exec(
+        path,
+        args=args_list,
+        env=env_dict,
+        chdir=cwd,
+        user=action_declaration.get("user", "root"),
+    )
+
+    if retcode not in action_declaration.get("accepted_return_codes", [0]):
+        raise MoulinetteError(retcode, "Error while executing action '%s' of app '%s': return code %s" % (action, app_id, retcode))
+
+    os.remove(path)
+
+    return logger.success("Action successed!")
+
+
+# Config panel todo list:
+# * docstrings
+# * merge translations on the json once the workflow is in place
+def app_config_show_panel(app_id):
+    logger.warning(m18n.n('experimental_feature'))
+
+    from yunohost.hook import hook_exec
+
+    # this will take care of checking if the app is installed
+    app_info_dict = app_info(app_id)
+
+    config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json')
+    config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config')
+
+    if not os.path.exists(config_panel) or not os.path.exists(config_script):
+        return {
+            "config_panel": [],
+        }
+
+    config_panel = read_json(config_panel)
+
+    env = {"YNH_APP_ID": app_id}
+    parsed_values = {}
+
+    # I need to parse stdout to communicate between scripts because I can't
+    # read the child environment :( (that would simplify things so much)
+    # after hours of research this is apparently quite a standard way, another
+    # option would be to add an explicite pipe or a named pipe for that
+    # a third option would be to write in a temporary file but I don't like
+    # that because that could expose sensitive data
+    def parse_stdout(line):
+        line = line.rstrip()
+        logger.info(line)
+
+        if line.strip().startswith("YNH_CONFIG_") and "=" in line:
+            # XXX error handling?
+            # XXX this might not work for multilines stuff :( (but echo without
+            # formatting should do it no?)
+            key, value = line.strip().split("=", 1)
+            logger.debug("config script declared: %s -> %s", key, value)
+            parsed_values[key] = value
+
+    return_code = hook_exec(config_script,
+              args=["show"],
+              env=env,
+              user="root",
+              stdout_callback=parse_stdout,
+    )
+
+    if return_code != 0:
+        raise Exception("script/config show return value code: %s (considered as an error)", return_code)
+
+    logger.debug("Generating global variables:")
+    for tab in config_panel.get("panel", []):
+        tab_id = tab["id"]  # this makes things easier to debug on crash
+        for section in tab.get("sections", []):
+            section_id = section["id"]
+            for option in section.get("options", []):
+                option_id = option["id"]
+                generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper()
+                option["id"] = generated_id
+                logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), generated_id)
+
+                if generated_id in parsed_values:
+                    # XXX we should probably uses the one of install here but it's at a POC state right now
+                    option_type = option["type"]
+                    if option_type == "bool":
+                        assert parsed_values[generated_id].lower() in ("true", "false")
+                        option["value"] = True if parsed_values[generated_id].lower() == "true" else False
+                    elif option_type == "integer":
+                        option["value"] = int(parsed_values[generated_id])
+                    elif option_type == "text":
+                        option["value"] = parsed_values[generated_id]
+                else:
+                    logger.debug("Variable '%s' is not declared by config script, using default", generated_id)
+                    option["value"] = option["default"]
+
+    return {
+        "app_id": app_id,
+        "app_name": app_info_dict["name"],
+        "config_panel": config_panel,
+    }
+
+
+def app_config_apply(app_id, args):
+    logger.warning(m18n.n('experimental_feature'))
+
+    from yunohost.hook import hook_exec
+
+    installed = _is_installed(app_id)
+    if not installed:
+        raise MoulinetteError(errno.ENOPKG,
+                              m18n.n('app_not_installed', app=app_id))
+
+    config_panel = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json')
+    config_script = os.path.join(APPS_SETTING_PATH, app_id, 'scripts', 'config')
+
+    if not os.path.exists(config_panel) or not os.path.exists(config_script):
+        # XXX real exception
+        raise Exception("Not config-panel.json nor scripts/config")
+
+    config_panel = read_json(config_panel)
+
+    env = {"YNH_APP_ID": app_id}
+    args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {}
+
+    for tab in config_panel.get("panel", []):
+        tab_id = tab["id"]  # this makes things easier to debug on crash
+        for section in tab.get("sections", []):
+            section_id = section["id"]
+            for option in section.get("options", []):
+                option_id = option["id"]
+                generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper()
+
+                if generated_id in args:
+                    logger.debug("include into env %s=%s", generated_id, args[generated_id])
+                    env[generated_id] = args[generated_id]
+                else:
+                    logger.debug("no value for key id %s", generated_id)
+
+    # for debug purpose
+    for key in args:
+        if key not in env:
+            logger.warning("Ignore key '%s' from arguments because it is not in the config", key)
+
+    return_code = hook_exec(config_script,
+              args=["apply"],
+              env=env,
+              user="root",
+    )
+
+    if return_code != 0:
+        raise Exception("'script/config apply' return value code: %s (considered as an error)", return_code)
+
+    logger.success("Config updated as expected")
+
+
 def _get_app_settings(app_id):
     """
     Get settings of an installed app
@@ -1877,143 +2090,183 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None):
         action -- The action to retrieve arguments for
         args -- A dictionnary of arguments to parse
 
+    """
+    if action not in manifest['arguments']:
+        logger.debug("no arguments found for '%s' in manifest", action)
+        return OrderedDict()
+
+    action_args = manifest['arguments'][action]
+    return _parse_action_args_in_yunohost_format(args, action_args, auth)
+
+
+def _parse_args_for_action(action, args={}, auth=None):
+    """Parse arguments needed for an action from the actions list
+
+    Retrieve specified arguments for the action from the manifest, and parse
+    given args according to that. If some required arguments are not provided,
+    its values will be asked if interaction is possible.
+    Parsed arguments will be returned as an OrderedDict
+
+    Keyword arguments:
+        action -- The action
+        args -- A dictionnary of arguments to parse
+
+    """
+    args_dict = OrderedDict()
+
+    if 'arguments' not in action:
+        logger.debug("no arguments found for '%s' in manifest", action)
+        return args_dict
+
+    action_args = action['arguments']
+
+    return _parse_action_args_in_yunohost_format(args, action_args, auth)
+
+
+def _parse_action_args_in_yunohost_format(args, action_args, auth=None):
+    """Parse arguments store in either manifest.json or actions.json
     """
     from yunohost.domain import (domain_list, _get_maindomain,
                                  domain_url_available, _normalize_domain_path)
-    from yunohost.user import user_info
+    from yunohost.user import user_info, user_list
 
     args_dict = OrderedDict()
-    try:
-        action_args = manifest['arguments'][action]
-    except KeyError:
-        logger.debug("no arguments found for '%s' in manifest", action)
-    else:
-        for arg in action_args:
-            arg_name = arg['name']
-            arg_type = arg.get('type', 'string')
-            arg_default = arg.get('default', None)
-            arg_choices = arg.get('choices', [])
-            arg_value = None
 
-            # Transpose default value for boolean type and set it to
+    for arg in action_args:
-            # false if not defined.
+        arg_name = arg['name']
-            if arg_type == 'boolean':
+        arg_type = arg.get('type', 'string')
-                arg_default = 1 if arg_default else 0
+        arg_default = arg.get('default', None)
+        arg_choices = arg.get('choices', [])
+        arg_value = None
 
-            # Attempt to retrieve argument value
+        # Transpose default value for boolean type and set it to
-            if arg_name in args:
+        # false if not defined.
-                arg_value = args[arg_name]
+        if arg_type == 'boolean':
-            else:
+            arg_default = 1 if arg_default else 0
-                if 'ask' in arg:
-                    # Retrieve proper ask string
-                    ask_string = _value_for_locale(arg['ask'])
 
-                    # Append extra strings
+        # Attempt to retrieve argument value
+        if arg_name in args:
+            arg_value = args[arg_name]
+        else:
+            if 'ask' in arg:
+                # Retrieve proper ask string
+                ask_string = _value_for_locale(arg['ask'])
+
+                # Append extra strings
+                if arg_type == 'boolean':
+                    ask_string += ' [yes | no]'
+                elif arg_choices:
+                    ask_string += ' [{0}]'.format(' | '.join(arg_choices))
+
+                if arg_default is not None:
                     if arg_type == 'boolean':
-                        ask_string += ' [0 | 1]'
+                        ask_string += ' (default: {0})'.format("yes" if arg_default == 1 else "no")
-                    elif arg_choices:
-                        ask_string += ' [{0}]'.format(' | '.join(arg_choices))
-                    if arg_default is not None:
-                        ask_string += ' (default: {0})'.format(arg_default)
-
-                    # Check for a password argument
-                    is_password = True if arg_type == 'password' else False
-
-                    if arg_type == 'domain':
-                        arg_default = _get_maindomain()
-                        ask_string += ' (default: {0})'.format(arg_default)
-                        msignals.display(m18n.n('domains_available'))
-                        for domain in domain_list(auth)['domains']:
-                            msignals.display("- {}".format(domain))
-
-                    try:
-                        input_string = msignals.prompt(ask_string, is_password)
-                    except NotImplementedError:
-                        input_string = None
-                    if (input_string == '' or input_string is None) \
-                            and arg_default is not None:
-                        arg_value = arg_default
                     else:
-                        arg_value = input_string
+                        ask_string += ' (default: {0})'.format(arg_default)
-                elif arg_default is not None:
-                    arg_value = arg_default
 
-            # Validate argument value
+                # Check for a password argument
-            if (arg_value is None or arg_value == '') \
+                is_password = True if arg_type == 'password' else False
-                    and not arg.get('optional', False):
-                raise MoulinetteError(errno.EINVAL,
-                    m18n.n('app_argument_required', name=arg_name))
-            elif arg_value is None:
-                args_dict[arg_name] = ''
-                continue
 
-            # Validate argument choice
+                if arg_type == 'domain':
-            if arg_choices and arg_value not in arg_choices:
+                    arg_default = _get_maindomain()
-                raise MoulinetteError(errno.EINVAL,
+                    ask_string += ' (default: {0})'.format(arg_default)
-                    m18n.n('app_argument_choice_invalid',
+                    msignals.display(m18n.n('domains_available'))
-                        name=arg_name, choices=', '.join(arg_choices)))
+                    for domain in domain_list(auth)['domains']:
+                        msignals.display("- {}".format(domain))
+
+                if arg_type == 'user':
+                    msignals.display(m18n.n('users_available'))
+                    for user in user_list(auth)['users'].keys():
+                        msignals.display("- {}".format(user))
 
-            # Validate argument type
-            if arg_type == 'domain':
-                if arg_value not in domain_list(auth)['domains']:
-                    raise MoulinetteError(errno.EINVAL,
-                        m18n.n('app_argument_invalid',
-                            name=arg_name, error=m18n.n('domain_unknown')))
-            elif arg_type == 'user':
                 try:
-                    user_info(auth, arg_value)
+                    input_string = msignals.prompt(ask_string, is_password)
-                except MoulinetteError as e:
+                except NotImplementedError:
-                    raise MoulinetteError(errno.EINVAL,
+                    input_string = None
-                        m18n.n('app_argument_invalid',
+                if (input_string == '' or input_string is None) \
-                            name=arg_name, error=e.strerror))
+                        and arg_default is not None:
-            elif arg_type == 'app':
+                    arg_value = arg_default
-                if not _is_installed(arg_value):
-                    raise MoulinetteError(errno.EINVAL,
-                        m18n.n('app_argument_invalid',
-                            name=arg_name, error=m18n.n('app_unknown')))
-            elif arg_type == 'boolean':
-                if isinstance(arg_value, bool):
-                    arg_value = 1 if arg_value else 0
                 else:
-                    try:
+                    arg_value = input_string
-                        arg_value = int(arg_value)
+            elif arg_default is not None:
-                        if arg_value not in [0, 1]:
+                arg_value = arg_default
-                            raise ValueError()
-                    except (TypeError, ValueError):
-                        raise MoulinetteError(errno.EINVAL,
-                            m18n.n('app_argument_choice_invalid',
-                                name=arg_name, choices='0, 1'))
-            args_dict[arg_name] = arg_value
 
-        # END loop over action_args...
+        # Validate argument value
+        if (arg_value is None or arg_value == '') \
+                and not arg.get('optional', False):
+            raise MoulinetteError(errno.EINVAL,
+                m18n.n('app_argument_required', name=arg_name))
+        elif arg_value is None:
+            args_dict[arg_name] = ''
+            continue
 
-        # If there's only one "domain" and "path", validate that domain/path
+        # Validate argument choice
-        # is an available url and normalize the path.
+        if arg_choices and arg_value not in arg_choices:
+            raise MoulinetteError(errno.EINVAL,
+                m18n.n('app_argument_choice_invalid',
+                    name=arg_name, choices=', '.join(arg_choices)))
 
-        domain_args = [arg["name"] for arg in action_args
+        # Validate argument type
-                       if arg.get("type", "string") == "domain"]
+        if arg_type == 'domain':
-        path_args = [arg["name"] for arg in action_args
+            if arg_value not in domain_list(auth)['domains']:
-                     if arg.get("type", "string") == "path"]
-
-        if len(domain_args) == 1 and len(path_args) == 1:
-
-            domain = args_dict[domain_args[0]]
-            path = args_dict[path_args[0]]
-            domain, path = _normalize_domain_path(domain, path)
-
-            # Check the url is available
-            if not domain_url_available(auth, domain, path):
                 raise MoulinetteError(errno.EINVAL,
-                                      m18n.n('app_location_unavailable'))
+                    m18n.n('app_argument_invalid',
+                        name=arg_name, error=m18n.n('domain_unknown')))
+        elif arg_type == 'user':
+            try:
+                user_info(auth, arg_value)
+            except MoulinetteError as e:
+                raise MoulinetteError(errno.EINVAL,
+                    m18n.n('app_argument_invalid',
+                        name=arg_name, error=e.strerror))
+        elif arg_type == 'app':
+            if not _is_installed(arg_value):
+                raise MoulinetteError(errno.EINVAL,
+                    m18n.n('app_argument_invalid',
+                        name=arg_name, error=m18n.n('app_unknown')))
+        elif arg_type == 'boolean':
+            if isinstance(arg_value, bool):
+                arg_value = 1 if arg_value else 0
+            else:
+                if str(arg_value).lower() in ["1", "yes", "y"]:
+                    arg_value = 1
+                elif str(arg_value).lower() in ["0", "no", "n"]:
+                    arg_value = 0
+                else:
+                    raise MoulinetteError(errno.EINVAL,
+                        m18n.n('app_argument_choice_invalid',
+                            name=arg_name, choices='yes, no, y, n, 1, 0'))
+        args_dict[arg_name] = arg_value
 
-            # (We save this normalized path so that the install script have a
+    # END loop over action_args...
-            # standard path format to deal with no matter what the user inputted)
+
-            args_dict[path_args[0]] = path
+    # If there's only one "domain" and "path", validate that domain/path
+    # is an available url and normalize the path.
+
+    domain_args = [arg["name"] for arg in action_args
+                   if arg.get("type", "string") == "domain"]
+    path_args = [arg["name"] for arg in action_args
+                 if arg.get("type", "string") == "path"]
+
+    if len(domain_args) == 1 and len(path_args) == 1:
+
+        domain = args_dict[domain_args[0]]
+        path = args_dict[path_args[0]]
+        domain, path = _normalize_domain_path(domain, path)
+
+        # Check the url is available
+        if not domain_url_available(auth, domain, path):
+            raise MoulinetteError(errno.EINVAL,
+                                  m18n.n('app_location_unavailable'))
+
+        # (We save this normalized path so that the install script have a
+        # standard path format to deal with no matter what the user inputted)
+        args_dict[path_args[0]] = path
 
     return args_dict
 
 
-def _make_environment_dict(args_dict):
+def _make_environment_dict(args_dict, prefix="APP_ARG_"):
     """
     Convert a dictionnary containing manifest arguments
     to a dictionnary of env. var. to be passed to scripts
@@ -2024,7 +2277,7 @@ def _make_environment_dict(args_dict):
     """
     env_dict = {}
     for arg_name, arg_value in args_dict.items():
-        env_dict["YNH_APP_ARG_%s" % arg_name.upper()] = arg_value
+        env_dict["YNH_%s%s" % (prefix, arg_name.upper())] = arg_value
     return env_dict
 
 

src/yunohost/certificate.py

@@ -40,7 +40,6 @@ from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
 
-import yunohost.domain
 from yunohost.utils.network import get_public_ip
 
 from moulinette import m18n
@@ -96,6 +95,8 @@ def certificate_status(auth, domain_list, full=False):
         full        -- Display more info about the certificates
     """
 
+    import yunohost.domain
+
     # Check if old letsencrypt_ynh is installed
     # TODO / FIXME - Remove this in the future once the letsencrypt app is
     # not used anymore
@@ -250,6 +251,8 @@ def _certificate_install_selfsigned(domain_list, force=False):
 
 
 def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=False, staging=False):
+    import yunohost.domain
+
     if not os.path.exists(ACCOUNT_KEY_FILE):
         _generate_account_key()
 
@@ -298,7 +301,7 @@ def _certificate_install_letsencrypt(auth, domain_list, force=False, no_checks=F
             uo.start()
 
             _configure_for_acme_challenge(auth, domain)
-            _fetch_and_enable_new_certificate(domain, staging)
+            _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
             _install_cron()
 
             logger.success(
@@ -323,6 +326,8 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
         email      -- Emails root if some renewing failed
     """
 
+    import yunohost.domain
+
     # Check if old letsencrypt_ynh is installed
     # TODO / FIXME - Remove this in the future once the letsencrypt app is
     # not used anymore
@@ -399,7 +404,7 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
 
             uo.start()
 
-            _fetch_and_enable_new_certificate(domain, staging)
+            _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
 
             logger.success(
                 m18n.n("certmanager_cert_renew_success", domain=domain))
@@ -426,6 +431,8 @@ def certificate_renew(auth, domain_list, force=False, no_checks=False, email=Fal
 ###############################################################################
 
 def _check_old_letsencrypt_app():
+    import yunohost.domain
+
     installedAppIds = [app["id"] for app in yunohost.app.app_list(installed=True)["apps"]]
 
     if "letsencrypt" not in installedAppIds:
@@ -538,7 +545,7 @@ def _check_acme_challenge_configuration(domain):
         return True
 
 
-def _fetch_and_enable_new_certificate(domain, staging=False):
+def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False):
     # Make sure tmp folder exists
     logger.debug("Making sure tmp folders exists...")
 
@@ -579,6 +586,7 @@ def _fetch_and_enable_new_certificate(domain, staging=False):
                                               domain_csr_file,
                                               WEBROOT_FOLDER,
                                               log=logger,
+                                              no_checks=no_checks,
                                               CA=certification_authority)
     except ValueError as e:
         if "urn:acme:error:rateLimited" in str(e):

src/yunohost/domain.py

@@ -114,7 +114,7 @@ def domain_add(uo, auth, domain, dyndns=False):
 
         # Don't regen these conf if we're still in postinstall
         if os.path.exists('/etc/yunohost/installed'):
-            service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'rmilter'])
+            service_regen_conf(names=['nginx', 'metronome', 'dnsmasq'])
             app_ssowatconf(auth)
 
     except Exception, e:

src/yunohost/hook.py

@@ -297,7 +297,8 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None,
 
 
 def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
-              chdir=None, env=None, user="admin"):
+              chdir=None, env=None, user="admin", stdout_callback=None,
+              stderr_callback=None):
     """
     Execute hook from a file with arguments
 
@@ -361,9 +362,10 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
 
     # Define output callbacks and call command
     callbacks = (
-        lambda l: logger.debug(l.rstrip()),
+        stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()),
-        lambda l: logger.warning(l.rstrip()),
+        stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()),
     )
+    logger.debug("About to run the command '%s'" % command)
     returncode = call_async_output(
         command, callbacks, shell=False, cwd=chdir
     )

src/yunohost/tools.py

@@ -239,14 +239,14 @@ def _is_inside_container():
     Returns True or False
     """
 
-    # See https://stackoverflow.com/a/37016302
+    # See https://www.2daygeek.com/check-linux-system-physical-virtual-machine-virtualization-technology/
-    p = subprocess.Popen("sudo cat /proc/1/sched".split(),
+    p = subprocess.Popen("sudo systemd-detect-virt".split(),
                          stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT)
 
     out, _ = p.communicate()
-
+    container = ['lxc','lxd','docker']
-    return out.split()[1] != "(1,"
+    return out.split()[0] in container
 
 
 @is_unit_operation()
@@ -336,7 +336,7 @@ def tools_postinstall(uo, domain, password, ignore_dyndns=False):
             ssowat_conf = json.loads(str(json_conf.read()))
     except ValueError as e:
         raise MoulinetteError(errno.EINVAL,
-                              m18n.n('ssowat_persistent_conf_read_error', error=e.strerror))
+                              m18n.n('ssowat_persistent_conf_read_error', error=str(e)))
     except IOError:
         ssowat_conf = {}
 
@@ -350,7 +350,7 @@ def tools_postinstall(uo, domain, password, ignore_dyndns=False):
             json.dump(ssowat_conf, f, sort_keys=True, indent=4)
     except IOError as e:
         raise MoulinetteError(errno.EPERM,
-                              m18n.n('ssowat_persistent_conf_write_error', error=e.strerror))
+                              m18n.n('ssowat_persistent_conf_write_error', error=str(e)))
 
     os.system('chmod 644 /etc/ssowat/conf.json.persistent')
 
@@ -415,6 +415,8 @@ def tools_postinstall(uo, domain, password, ignore_dyndns=False):
     service_regen_conf(force=True)
     logger.success(m18n.n('yunohost_configured'))
 
+    logger.warning(m18n.n('recommend_to_add_first_user'))
+
 
 def tools_update(ignore_apps=False, ignore_packages=False):
     """

src/yunohost/user.py

@@ -36,7 +36,6 @@ import subprocess
 from moulinette import m18n
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_file
 from yunohost.service import service_status
 from yunohost.log import is_unit_operation
 
@@ -114,7 +113,6 @@ def user_create(uo, auth, username, firstname, lastname, mail, password,
         mailbox_quota -- Mailbox size quota
 
     """
-    import pwd
     from yunohost.domain import domain_list, _get_maindomain
     from yunohost.hook import hook_callback
     from yunohost.app import app_ssowatconf

src/yunohost/utils/network.py

@@ -21,7 +21,7 @@
 import logging
 import re
 import subprocess
-from urllib import urlopen
+from moulinette.utils.network import download_text
 
 logger = logging.getLogger('yunohost.utils.network')
 
@@ -37,8 +37,9 @@ def get_public_ip(protocol=4):
         raise ValueError("invalid protocol version")
 
     try:
-        return urlopen(url).read().strip()
+        return download_text(url, timeout=30).strip()
-    except IOError:
+    except Exception as e:
+        logger.debug("Could not get public IPv%s : %s" % (str(protocol), str(e)))
         return None
 
 

src/yunohost/vendor/acme_tiny/acme_tiny.py

@@ -12,7 +12,7 @@ LOGGER = logging.getLogger(__name__)
 LOGGER.addHandler(logging.StreamHandler())
 LOGGER.setLevel(logging.INFO)
 
-def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA):
+def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, no_checks=False):
     # helper function base64 encode for jose spec
     def _b64(b):
         return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "")
@@ -111,16 +111,17 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA):
         with open(wellknown_path, "w") as wellknown_file:
             wellknown_file.write(keyauthorization)
 
-        # check that the file is in place
+        if not no_checks:  # sometime the local g
-        wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token)
+            # check that the file is in place
-        try:
+            wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token)
-            resp = urlopen(wellknown_url)
+            try:
-            resp_data = resp.read().decode('utf8').strip()
+                resp = urlopen(wellknown_url)
-            assert resp_data == keyauthorization
+                resp_data = resp.read().decode('utf8').strip()
-        except (IOError, AssertionError):
+                assert resp_data == keyauthorization
-            os.remove(wellknown_path)
+            except (IOError, AssertionError):
-            raise ValueError("Wrote file to {0}, but couldn't download {1}".format(
+                os.remove(wellknown_path)
-                wellknown_path, wellknown_url))
+                raise ValueError("Wrote file to {0}, but couldn't download {1}".format(
+                    wellknown_path, wellknown_url))
 
         # notify challenge are met
         code, result = _send_signed_request(challenge['uri'], {