Merge pull request #987 from YunoHost/enh-config-panel-file

[enh] New config-panel mechanism

.gitlab/ci/test.gitlab-ci.yml

@@ -112,14 +112,24 @@ test-appurl:
     changes:
       - src/yunohost/app.py
 
-test-apps-arguments-parsing:
+test-questions:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python3 -m pytest tests/test_apps_arguments_parsing.py
+    - python3 -m pytest tests/test_questions.py
+  only:
+    changes:
+      - src/yunohost/utils/config.py
+
+test-app-config:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - python3 -m pytest tests/test_app_config.py
   only:
     changes:
       - src/yunohost/app.py
+      - src/yunohost/utils/config.py
 
 test-changeurl:
   extends: .test-stage

data/actionsmap/yunohost.yml

@@ -853,24 +853,45 @@ app:
           subcategory_help: Applications configuration panel
           actions:
 
-              ### app_config_show_panel()
+              ### app_config_get()
-              show-panel:
+              get:
-                  action_help: show config panel for the application
+                  action_help: Display an app configuration
                   api: GET /apps/<app>/config-panel
                   arguments:
-                      app:
+                    app:
-                          help: App name
+                        help: App name
+                    key:
+                        help: A specific panel, section or a question identifier
+                        nargs: '?'
+                    -f:
+                        full: --full
+                        help: Display all details (meant to be used by the API)
+                        action: store_true
+                    -e:
+                        full: --export
+                        help: Only export key/values, meant to be reimported using "config set --args-file"
+                        action: store_true
 
-              ### app_config_apply()
+              ### app_config_set()
-              apply:
+              set:
-                  action_help: apply the new configuration
+                  action_help: Apply a new configuration
                   api: PUT /apps/<app>/config
                   arguments:
-                      app:
+                    app:
-                          help: App name
+                        help: App name
-                      -a:
+                    key:
-                          full: --args
+                        help: The question or panel key 
-                          help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path")
+                        nargs: '?'
+                    -v:
+                        full: --value
+                        help: new value
+                    -a:
+                        full: --args
+                        help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path")
+                    -f:
+                        full: --args-file
+                        help: YAML or JSON file with key/value couples
+                        type: open
 
 #############################
 #          Backup           #

data/helpers.d/backup

@@ -326,12 +326,25 @@ ynh_bind_or_cp() {
 ynh_store_file_checksum () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    local -A args_array=( [f]=file= )
+    local -A args_array=( [f]=file= [u]=update_only )
     local file
+    local update_only
+    update_only="${update_only:-0}"
+
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
     local checksum_setting_name=checksum_${file//[\/ ]/_}    # Replace all '/' and ' ' by '_'
+    
+    # If update only, we don't save the new checksum if no old checksum exist
+    if [ $update_only -eq 1 ] ; then
+        local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
+        if [ -z "${checksum_value}" ] ; then
+            unset backup_file_checksum
+            return 0
+        fi
+    fi
+    
     ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
 
     # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup

data/helpers.d/config

@@ -0,0 +1,325 @@
+#!/bin/bash
+
+
+_ynh_app_config_get() {
+    # From settings
+    local lines
+    lines=$(python3 << EOL
+import toml
+from collections import OrderedDict
+with open("../config_panel.toml", "r") as f:
+    file_content = f.read()
+loaded_toml = toml.loads(file_content, _dict=OrderedDict)
+
+for panel_name, panel in loaded_toml.items():
+    if not isinstance(panel, dict): continue
+    for section_name, section in panel.items():
+        if not isinstance(section, dict): continue
+        for name, param in section.items():
+            if not isinstance(param, dict):
+                continue
+            print(';'.join([
+                name,
+                param.get('type', 'string'),
+                param.get('bind', 'settings' if param.get('type', 'string') != 'file' else 'null')
+                ]))
+EOL
+)
+    for line in $lines
+    do
+        # Split line into short_setting, type and bind
+        IFS=';' read short_setting type bind <<< "$line"
+        local getter="get__${short_setting}"
+        binds[${short_setting}]="$bind"
+        types[${short_setting}]="$type"
+        file_hash[${short_setting}]=""
+        formats[${short_setting}]=""
+        # Get value from getter if exists
+        if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null;
+        then
+            old[$short_setting]="$($getter)"
+            formats[${short_setting}]="yaml"
+
+        elif [[ "$bind" == "null" ]]
+        then
+            old[$short_setting]="YNH_NULL"
+
+        # Get value from app settings or from another file
+        elif [[ "$type" == "file" ]]
+        then
+            if [[ "$bind" == "settings" ]]
+            then
+                ynh_die --message="File '${short_setting}' can't be stored in settings"
+            fi
+            old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2> /dev/null || echo YNH_NULL)"
+            file_hash[$short_setting]="true"
+
+        # Get multiline text from settings or from a full file
+        elif [[ "$type" == "text" ]]
+        then
+            if [[ "$bind" == "settings" ]]
+            then
+                old[$short_setting]="$(ynh_app_setting_get $app $short_setting)"
+            elif [[ "$bind" == *":"* ]]
+            then
+                ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+            else
+                old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)"
+            fi
+
+        # Get value from a kind of key/value file
+        else
+            local bind_after=""
+            if [[ "$bind" == "settings" ]]
+            then
+                bind=":/etc/yunohost/apps/$app/settings.yml"
+            fi
+            local bind_key="$(echo "$bind" | cut -d: -f1)"
+            bind_key=${bind_key:-$short_setting}
+            if [[ "$bind_key" == *">"* ]];
+            then
+                bind_after="$(echo "${bind_key}" | cut -d'>' -f1)"
+                bind_key="$(echo "${bind_key}" | cut -d'>' -f2)"
+            fi
+            local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+            old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}" --after="${bind_after}")"
+
+        fi
+    done
+
+
+}
+
+_ynh_app_config_apply() {
+    for short_setting in "${!old[@]}"
+    do
+        local setter="set__${short_setting}"
+        local bind="${binds[$short_setting]}"
+        local type="${types[$short_setting]}"
+        if [ "${changed[$short_setting]}" == "true" ]
+        then
+            # Apply setter if exists
+            if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null;
+            then
+                $setter
+
+            elif [[ "$bind" == "null" ]]
+            then
+                continue
+
+            # Save in a file
+            elif [[ "$type" == "file" ]]
+            then
+                if [[ "$bind" == "settings" ]]
+                then
+                    ynh_die --message="File '${short_setting}' can't be stored in settings"
+                fi
+                local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+                if [[ "${!short_setting}" == "" ]]
+                then
+                    ynh_backup_if_checksum_is_different --file="$bind_file"
+                    ynh_secure_remove --file="$bind_file"
+                    ynh_delete_file_checksum --file="$bind_file" --update_only
+                    ynh_print_info --message="File '$bind_file' removed"
+                else
+                    ynh_backup_if_checksum_is_different --file="$bind_file"
+                    cp "${!short_setting}" "$bind_file"
+                    ynh_store_file_checksum --file="$bind_file" --update_only
+                    ynh_print_info --message="File '$bind_file' overwrited with ${!short_setting}"
+                fi
+
+            # Save value in app settings
+            elif [[ "$bind" == "settings" ]]
+            then
+                ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}"
+                ynh_print_info --message="Configuration key '$short_setting' edited in app settings"
+
+            # Save multiline text in a file
+            elif [[ "$type" == "text" ]]
+            then
+                if [[ "$bind" == *":"* ]]
+                then
+                    ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+                fi
+                local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+                ynh_backup_if_checksum_is_different --file="$bind_file"
+                echo "${!short_setting}" > "$bind_file"
+                ynh_store_file_checksum --file="$bind_file" --update_only
+                ynh_print_info --message="File '$bind_file' overwrited with the content you provieded in '${short_setting}' question"
+
+            # Set value into a kind of key/value file
+            else
+                local bind_after=""
+                local bind_key="$(echo "$bind" | cut -d: -f1)"
+                bind_key=${bind_key:-$short_setting}
+                if [[ "$bind_key" == *">"* ]];
+                then
+                    bind_after="$(echo "${bind_key}" | cut -d'>' -f1)"
+                    bind_key="$(echo "${bind_key}" | cut -d'>' -f2)"
+                fi
+                local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+
+                ynh_backup_if_checksum_is_different --file="$bind_file"
+                ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" --after="${bind_after}"
+                ynh_store_file_checksum --file="$bind_file" --update_only
+
+                # We stored the info in settings in order to be able to upgrade the app
+                ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}"
+                ynh_print_info --message="Configuration key '$bind_key' edited into $bind_file"
+
+            fi
+        fi
+    done
+}
+
+_ynh_app_config_show() {
+    for short_setting in "${!old[@]}"
+    do
+        if [[ "${old[$short_setting]}" != YNH_NULL ]]
+        then
+            if [[ "${formats[$short_setting]}" == "yaml" ]]
+            then
+                ynh_return "${short_setting}:"
+                ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/  /g')"
+            else
+                ynh_return "${short_setting}: "'"'"$(echo "${old[$short_setting]}" | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\n\n/g')"'"'
+
+            fi
+        fi
+    done
+}
+
+_ynh_app_config_validate() {
+    # Change detection
+    ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1
+    local nothing_changed=true
+    local changes_validated=true
+    for short_setting in "${!old[@]}"
+    do
+        changed[$short_setting]=false
+        if [ -z ${!short_setting+x} ]
+        then
+            # Assign the var with the old value in order to allows multiple
+            # args validation
+            declare "$short_setting"="${old[$short_setting]}"
+            continue
+        fi
+        if [ ! -z "${file_hash[${short_setting}]}" ]
+        then
+            file_hash[old__$short_setting]=""
+            file_hash[new__$short_setting]=""
+            if [ -f "${old[$short_setting]}" ]
+            then
+                file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
+                if [ -z "${!short_setting}" ]
+                then
+                    changed[$short_setting]=true
+                    nothing_changed=false
+                fi
+            fi
+            if [ -f "${!short_setting}" ]
+            then
+                file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1)
+                if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
+                then
+                    changed[$short_setting]=true
+                    nothing_changed=false
+                fi
+            fi
+        else
+            if [[ "${!short_setting}" != "${old[$short_setting]}" ]]
+            then
+                changed[$short_setting]=true
+                nothing_changed=false
+            fi
+        fi
+    done
+    if [[ "$nothing_changed" == "true" ]]
+    then
+        ynh_print_info --message="Nothing has changed"
+        exit 0
+    fi
+
+    # Run validation if something is changed
+    ynh_script_progression --message="Validating the new configuration..." --weight=1
+
+    for short_setting in "${!old[@]}"
+    do
+        [[ "${changed[$short_setting]}" == "false" ]] && continue
+        local result=""
+        if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null;
+        then
+            result="$(validate__$short_setting)"
+        fi
+        if [ -n "$result" ]
+        then
+            #
+            # Return a yaml such as:
+            #
+            # validation_errors:
+            #   some_key: "An error message"
+            #   some_other_key: "Another error message"
+            #
+            # We use changes_validated to know if this is
+            # the first validation error
+            if [[ "$changes_validated" == true ]]
+            then
+                ynh_return "validation_errors:"
+            fi
+            ynh_return "  ${short_setting}: \"$result\""
+            changes_validated=false
+        fi
+    done
+
+    # If validation failed, exit the script right now (instead of going into apply)
+    # Yunohost core will pick up the errors returned via ynh_return previously
+    if [[ "$changes_validated" == "false" ]]
+    then
+        exit 0
+    fi
+
+}
+
+ynh_app_config_get() {
+    _ynh_app_config_get
+}
+
+ynh_app_config_show() {
+    _ynh_app_config_show
+}
+
+ynh_app_config_validate() {
+    _ynh_app_config_validate
+}
+
+ynh_app_config_apply() {
+    _ynh_app_config_apply
+}
+
+ynh_app_config_run() {
+    declare -Ag old=()
+    declare -Ag changed=()
+    declare -Ag file_hash=()
+    declare -Ag binds=()
+    declare -Ag types=()
+    declare -Ag formats=()
+
+    case $1 in
+        show)
+            ynh_app_config_get
+            ynh_app_config_show
+            ;;
+        apply)
+            max_progression=4
+            ynh_script_progression --message="Reading config panel description and current configuration..."
+            ynh_app_config_get
+
+            ynh_app_config_validate
+
+            ynh_script_progression --message="Applying the new configuration..."
+            ynh_app_config_apply
+            ynh_script_progression --message="Configuration of $app completed" --last
+            ;;
+    esac
+}
+

data/helpers.d/utils

@@ -473,6 +473,197 @@ ynh_replace_vars () {
     done
 }
 
+# Get a value from heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_read_var_in_file --file=PATH --key=KEY
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to get
+#
+# This helpers match several var affectation use case in several languages
+# We don't use jq or equivalent to keep comments and blank space in files
+# This helpers work line by line, it is not able to work correctly
+# if you have several identical keys in your files
+#
+# Example of line this helpers can managed correctly
+# .yml
+#     title: YunoHost documentation
+#     email: 'yunohost@yunohost.org'
+# .json
+#     "theme": "colib'ris",
+#     "port": 8102
+#     "some_boolean":     false,
+#     "user": null
+# .ini
+#     some_boolean = On
+#     action = "Clear"
+#     port = 20
+# .php
+#     $user=
+#     user => 20
+# .py
+#     USER = 8102
+#     user = 'https://donate.local'
+#     CUSTOM['user'] = 'YunoHost'
+#
+# Requires YunoHost version 4.3 or higher.
+ynh_read_var_in_file() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fka
+    local -A args_array=( [f]=file= [k]=key= [a]=after=)
+    local file
+    local key
+    local after
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    after="${after:-}"
+
+    [[ -f $file ]] || ynh_die --message="File $file does not exists"
+
+    # Get the line number after which we search for the variable
+    local line_number=1
+    if [[ -n "$after" ]];
+    then
+        line_number=$(grep -n $after $file | cut -d: -f1)
+        if [[ -z "$line_number" ]];
+        then
+            return 1
+        fi
+    fi
+
+    local filename="$(basename -- "$file")"
+    local ext="${filename##*.}"
+    local endline=',;'
+    local assign="=>|:|="
+    local comments="#"
+    local string="\"'"
+    if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then
+        endline='#'
+    fi
+    if [[ "$ext" =~ ^ini|env$ ]]; then
+        comments="[;#]"
+    fi
+    if [[ "php"  == "$ext" ]] || [[ "$ext" == "js" ]]; then
+        comments="//"
+    fi
+    local list='\[\s*['$string']?\w+['$string']?\]'
+    local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*'
+    var_part+="[$string]?${key}[$string]?"
+    var_part+='\s*\]?\s*'
+    var_part+="($assign)"
+    var_part+='\s*'
+
+    # Extract the part after assignation sign
+    local expression_with_comment="$(tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL | head -n1)"
+    if [[ "$expression_with_comment" == "YNH_NULL" ]]; then
+        echo YNH_NULL
+        return 0
+    fi
+
+    # Remove comments if needed
+    local expression="$(echo "$expression_with_comment" | sed "s@$comments[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")"
+
+    local first_char="${expression:0:1}"
+    if [[ "$first_char" == '"' ]] ; then
+        echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1  | sed 's/\\"/"/g'
+    elif [[ "$first_char" == "'" ]] ; then
+        echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1  | sed "s/\\\\'/'/g"
+    else
+        echo "$expression"
+    fi
+}
+
+# Set a value into heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to set
+# | arg: -v, --value=     - the value to set
+#
+# Requires YunoHost version 4.3 or higher.
+ynh_write_var_in_file() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fkva
+    local -A args_array=( [f]=file= [k]=key= [v]=value= [a]=after=)
+    local file
+    local key
+    local value
+    local after
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    after="${after:-}"
+
+    [[ -f $file ]] || ynh_die --message="File $file does not exists"
+
+    # Get the line number after which we search for the variable
+    local line_number=1
+    if [[ -n "$after" ]];
+    then
+        line_number=$(grep -n $after $file | cut -d: -f1)
+        if [[ -z "$line_number" ]];
+        then
+            return 1
+        fi
+    fi
+    local range="${line_number},\$ "
+
+    local filename="$(basename -- "$file")"
+    local ext="${filename##*.}"
+    local endline=',;'
+    local assign="=>|:|="
+    local comments="#"
+    local string="\"'"
+    if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then
+        endline='#'
+    fi
+    if [[ "$ext" =~ ^ini|env$ ]]; then
+        comments="[;#]"
+    fi
+    if [[ "php"  == "$ext" ]] || [[ "$ext" == "js" ]]; then
+        comments="//"
+    fi
+    local list='\[\s*['$string']?\w+['$string']?\]'
+    local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*'
+    var_part+="[$string]?${key}[$string]?"
+    var_part+='\s*\]?\s*'
+    var_part+="($assign)"
+    var_part+='\s*'
+
+    # Extract the part after assignation sign
+    local expression_with_comment="$(tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL | head -n1)"
+    if [[ "$expression_with_comment" == "YNH_NULL" ]]; then
+        return 1
+    fi
+    
+    # Remove comments if needed
+    local expression="$(echo "$expression_with_comment" | sed "s@$comments[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")"
+    endline=${expression_with_comment#"$expression"}
+    endline="$(echo "$endline" | sed 's/\\/\\\\/g')"
+    value="$(echo "$value" | sed 's/\\/\\\\/g')"
+    local first_char="${expression:0:1}"
+    delimiter=$'\001'
+    if [[ "$first_char" == '"' ]] ; then
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # So we need \\\\ to go through 2 sed
+        value="$(echo "$value" | sed 's/"/\\\\"/g')"
+        sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file}
+    elif [[ "$first_char" == "'" ]] ; then
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # However double quotes implies to double \\ to
+        # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str
+        value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")"
+        sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file}
+    else
+        if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]] ; then
+            value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"'
+        fi
+        if [[ "$ext" =~ ^yaml|yml$ ]] ; then
+            value=" $value"
+        fi
+        sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file}
+    fi
+}
+
+
 # Render templates with Jinja2
 #
 # [internal]

locales/en.json

@@ -13,14 +13,15 @@
     "app_already_installed": "{app} is already installed",
     "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.",
     "app_already_up_to_date": "{app} is already up-to-date",
-    "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}'",
+    "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'",
     "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}",
     "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason",
     "app_argument_required": "Argument '{name}' is required",
-    "app_change_url_failed_nginx_reload": "Could not reload NGINX. Here is the output of 'nginx -t':\n{nginx_errors}",
     "app_change_url_identical_domains": "The old and new domain/url_path are identical ('{domain}{path}'), nothing to do.",
     "app_change_url_no_script": "The app '{app_name}' doesn't support URL modification yet. Maybe you should upgrade it.",
     "app_change_url_success": "{app} URL is now {domain}{path}",
+    "app_config_unable_to_apply": "Failed to apply config panel values.",
+    "app_config_unable_to_read": "Failed to read config panel values.",
     "app_extraction_failed": "Could not extract the installation files",
     "app_full_domain_unavailable": "Sorry, this app must be installed on a domain of its own, but other apps are already installed on the domain '{domain}'. You could use a subdomain dedicated to this app instead.",
     "app_id_invalid": "Invalid app ID",
@@ -139,10 +140,22 @@
     "certmanager_self_ca_conf_file_not_found": "Could not find configuration file for self-signing authority (file: {file})",
     "certmanager_unable_to_parse_self_CA_name": "Could not parse name of self-signing authority (file: {file})",
     "certmanager_warning_subdomain_dns_record": "Subdomain '{subdomain}' does not resolve to the same IP address as '{domain}'. Some features will not be available until you fix this and regenerate the certificate.",
+    "config_apply_failed": "Applying the new configuration failed: {error}",
+    "config_cant_set_value_on_section": "You can't set a single value on an entire config section.",
+    "config_forbidden_keyword": "The keyword '{keyword}' is reserved, you can't create or use a config panel with a question with this id.",
+    "config_no_panel": "No config panel found.",
+    "config_unknown_filter_key": "The filter key '{filter_key}' is incorrect.",
+    "config_validate_color": "Should be a valid RGB hexadecimal color",
+    "config_validate_date": "Should be a valid date like in the format YYYY-MM-DD",
+    "config_validate_email": "Should be a valid email",
+    "config_validate_time": "Should be a valid time like XX:YY",
+    "config_validate_url": "Should be a valid web URL",
+    "config_version_not_supported": "Config panel versions '{version}' are not supported.",
     "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'",
     "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'",
     "confirm_app_install_warning": "Warning: This app may work, but is not well-integrated in YunoHost. Some features such as single sign-on and backup/restore might not be available. Install anyway? [{answers}] ",
     "custom_app_url_required": "You must provide a URL to upgrade your custom app {app}",
+    "danger": "Danger:",
     "diagnosis_apps_allgood": "All installed apps respect basic packaging practices",
     "diagnosis_apps_bad_quality": "This application is currently flagged as broken on YunoHost's application catalog. This may be a temporary issue while the maintainers attempt to fix the issue. In the meantime, upgrading this app is disabled.",
     "diagnosis_apps_broken": "This application is currently flagged as broken on YunoHost's application catalog. This may be a temporary issue while the maintainers attempt to fix the issue. In the meantime, upgrading this app is disabled.",
@@ -302,7 +315,6 @@
     "domain_name_unknown": "Domain '{domain}' unknown",
     "domain_remove_confirm_apps_removal": "Removing this domain will remove those applications:\n{apps}\n\nAre you sure you want to do that? [{answers}]",
     "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal",
-    "domain_unknown": "Unknown domain",
     "domains_available": "Available domains:",
     "done": "Done",
     "downloading": "Downloading...",
@@ -324,6 +336,7 @@
     "extracting": "Extracting...",
     "field_invalid": "Invalid field '{}'",
     "file_does_not_exist": "The file {path} does not exist.",
+    "file_extension_not_accepted": "Refusing file '{path}' because its extension is not among the accepted extensions: {accept}",
     "firewall_reload_failed": "Could not reload the firewall",
     "firewall_reloaded": "Firewall reloaded",
     "firewall_rules_cmd_failed": "Some firewall rule commands have failed. More info in log.",
@@ -337,8 +350,8 @@
     "global_settings_setting_backup_compress_tar_archives": "When creating new backups, compress the archives (.tar.gz) instead of uncompressed archives (.tar). N.B. : enabling this option means create lighter backup archives, but the initial backup procedure will be significantly longer and heavy on CPU.",
     "global_settings_setting_pop3_enabled": "Enable the POP3 protocol for the mail server",
     "global_settings_setting_security_experimental_enabled": "Enable experimental security features (don't enable this if you don't know what you're doing!)",
-    "global_settings_setting_security_nginx_redirect_to_https": "Redirect HTTP requests to HTTPs by default (DO NOT TURN OFF unless you really know what you're doing!)",
     "global_settings_setting_security_nginx_compatibility": "Compatibility vs. security tradeoff for the web server NGINX. Affects the ciphers (and other security-related aspects)",
+    "global_settings_setting_security_nginx_redirect_to_https": "Redirect HTTP requests to HTTPs by default (DO NOT TURN OFF unless you really know what you're doing!)",
     "global_settings_setting_security_password_admin_strength": "Admin password strength",
     "global_settings_setting_security_password_user_strength": "User password strength",
     "global_settings_setting_security_postfix_compatibility": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)",
@@ -380,6 +393,8 @@
     "hook_name_unknown": "Unknown hook name '{name}'",
     "installation_complete": "Installation completed",
     "invalid_number": "Must be a number",
+    "invalid_number_min": "Must be greater than {min}",
+    "invalid_number_max": "Must be lesser than {max}",
     "invalid_password": "Invalid password",
     "invalid_regex": "Invalid regex:'{regex}'",
     "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it",
@@ -388,8 +403,7 @@
     "ldap_server_is_down_restart_it": "The LDAP service is down, attempt to restart it...",
     "log_app_action_run": "Run action of the '{}' app",
     "log_app_change_url": "Change the URL of the '{}' app",
-    "log_app_config_apply": "Apply config to the '{}' app",
+    "log_app_config_set": "Apply config to the '{}' app",
-    "log_app_config_show_panel": "Show the config panel of the '{}' app",
     "log_app_install": "Install the '{}' app",
     "log_app_makedefault": "Make '{}' the default app",
     "log_app_remove": "Remove the '{}' app",
@@ -595,6 +609,7 @@
     "service_disabled": "The service '{service}' will not be started anymore when system boots.",
     "service_enable_failed": "Could not make the service '{service}' automatically start at boot.\n\nRecent service logs:{logs}",
     "service_enabled": "The service '{service}' will now be automatically started during system boots.",
+    "service_not_reloading_because_conf_broken": "Not reloading/restarting service '{name}' because it configuration is broken: {errors}",
     "service_regen_conf_is_deprecated": "'yunohost service regen-conf' is deprecated! Please use 'yunohost tools regen-conf' instead.",
     "service_reload_failed": "Could not reload the service '{service}'\n\nRecent service logs:{logs}",
     "service_reload_or_restart_failed": "Could not reload or restart the service '{service}'\n\nRecent service logs:{logs}",

src/yunohost/app.py

@@ -52,8 +52,13 @@ from moulinette.utils.filesystem import (
     mkdir,
 )
 
-from yunohost.service import service_status, _run_service_command
 from yunohost.utils import packages
+from yunohost.utils.config import (
+    ConfigPanel,
+    parse_args_in_yunohost_format,
+    Question,
+)
+from yunohost.utils.i18n import _value_for_locale
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.filesystem import free_space_in_directory
 from yunohost.log import is_unit_operation, OperationLogger
@@ -189,10 +194,7 @@ def app_info(app, full=False):
     """
     from yunohost.permission import user_permission_list
 
-    if not _is_installed(app):
+    _assert_is_installed(app)
-        raise YunohostValidationError(
-            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
-        )
 
     setting_path = os.path.join(APPS_SETTING_PATH, app)
     local_manifest = _get_manifest_of_app(setting_path)
@@ -421,6 +423,7 @@ def app_change_url(operation_logger, app, domain, path):
 
     """
     from yunohost.hook import hook_exec, hook_callback
+    from yunohost.service import service_reload_or_restart
 
     installed = _is_installed(app)
     if not installed:
@@ -489,15 +492,7 @@ def app_change_url(operation_logger, app, domain, path):
 
     app_ssowatconf()
 
-    # avoid common mistakes
+    service_reload_or_restart("nginx")
-    if _run_service_command("reload", "nginx") is False:
-        # grab nginx errors
-        # the "exit 0" is here to avoid check_output to fail because 'nginx -t'
-        # will return != 0 since we are in a failed state
-        nginx_errors = check_output("nginx -t; exit 0")
-        raise YunohostError(
-            "app_change_url_failed_nginx_reload", nginx_errors=nginx_errors
-        )
 
     logger.success(m18n.n("app_change_url_success", app=app, domain=domain, path=path))
 
@@ -516,7 +511,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
 
     """
     from packaging import version
-    from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
+    from yunohost.hook import hook_add, hook_remove, hook_callback, hook_exec_with_script_debug_if_failure
     from yunohost.permission import permission_sync_to_user
     from yunohost.regenconf import manually_modified_files
 
@@ -536,10 +531,8 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
     apps = [app_ for i, app_ in enumerate(apps) if app_ not in apps[:i]]
 
     # Abort if any of those app is in fact not installed..
-    for app in [app_ for app_ in apps if not _is_installed(app_)]:
+    for app_ in apps:
-        raise YunohostValidationError(
+        _assert_is_installed(app_)
-            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
-        )
 
     if len(apps) == 0:
         raise YunohostValidationError("apps_already_up_to_date")
@@ -640,36 +633,13 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
         # Execute the app upgrade script
         upgrade_failed = True
         try:
-            upgrade_retcode = hook_exec(
+            upgrade_failed, failure_message_with_debug_instructions = hook_exec_with_script_debug_if_failure(
-                extracted_app_folder + "/scripts/upgrade", env=env_dict
+                extracted_app_folder + "/scripts/upgrade",
-            )[0]
+                env=env_dict,
-
+                operation_logger=operation_logger,
-            upgrade_failed = True if upgrade_retcode != 0 else False
+                error_message_if_script_failed=m18n.n("app_upgrade_script_failed"),
-            if upgrade_failed:
+                error_message_if_failed=lambda e: m18n.n("app_upgrade_failed", app=app_instance_name, error=e)
-                error = m18n.n("app_upgrade_script_failed")
-                logger.error(
-                    m18n.n("app_upgrade_failed", app=app_instance_name, error=error)
-                )
-                failure_message_with_debug_instructions = operation_logger.error(error)
-                if Moulinette.interface.type != "api":
-                    dump_app_log_extract_for_debugging(operation_logger)
-        # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception
-        except (KeyboardInterrupt, EOFError):
-            upgrade_retcode = -1
-            error = m18n.n("operation_interrupted")
-            logger.error(
-                m18n.n("app_upgrade_failed", app=app_instance_name, error=error)
             )
-            failure_message_with_debug_instructions = operation_logger.error(error)
-        # Something wrong happened in Yunohost's code (most probably hook_exec)
-        except Exception:
-            import traceback
-
-            error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
-            logger.error(
-                m18n.n("app_upgrade_failed", app=app_instance_name, error=error)
-            )
-            failure_message_with_debug_instructions = operation_logger.error(error)
         finally:
             # Whatever happened (install success or failure) we check if it broke the system
             # and warn the user about it
@@ -699,7 +669,7 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
             if upgrade_failed or broke_the_system:
 
                 # display this if there are remaining apps
-                if apps[number + 1 :]:
+                if apps[number + 1:]:
                     not_upgraded_apps = apps[number:]
                     logger.error(
                         m18n.n(
@@ -753,7 +723,6 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False
             for file_to_copy in [
                 "actions.json",
                 "actions.toml",
-                "config_panel.json",
                 "config_panel.toml",
                 "conf",
             ]:
@@ -816,7 +785,7 @@ def app_install(
         force -- Do not ask for confirmation when installing experimental / low-quality apps
     """
 
-    from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback
+    from yunohost.hook import hook_add, hook_remove, hook_callback, hook_exec, hook_exec_with_script_debug_if_failure
     from yunohost.log import OperationLogger
     from yunohost.permission import (
         user_permission_list,
@@ -973,7 +942,6 @@ def app_install(
     for file_to_copy in [
         "actions.json",
         "actions.toml",
-        "config_panel.json",
         "config_panel.toml",
         "conf",
     ]:
@@ -1008,29 +976,13 @@ def app_install(
     # Execute the app install script
     install_failed = True
     try:
-        install_retcode = hook_exec(
+        install_failed, failure_message_with_debug_instructions = hook_exec_with_script_debug_if_failure(
-            os.path.join(extracted_app_folder, "scripts/install"), env=env_dict
+            os.path.join(extracted_app_folder, "scripts/install"),
-        )[0]
+            env=env_dict,
-        # "Common" app install failure : the script failed and returned exit code != 0
+            operation_logger=operation_logger,
-        install_failed = True if install_retcode != 0 else False
+            error_message_if_script_failed=m18n.n("app_install_script_failed"),
-        if install_failed:
+            error_message_if_failed=lambda e: m18n.n("app_install_failed", app=app_id, error=e)
-            error = m18n.n("app_install_script_failed")
+        )
-            logger.error(m18n.n("app_install_failed", app=app_id, error=error))
-            failure_message_with_debug_instructions = operation_logger.error(error)
-            if Moulinette.interface.type != "api":
-                dump_app_log_extract_for_debugging(operation_logger)
-    # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception
-    except (KeyboardInterrupt, EOFError):
-        error = m18n.n("operation_interrupted")
-        logger.error(m18n.n("app_install_failed", app=app_id, error=error))
-        failure_message_with_debug_instructions = operation_logger.error(error)
-    # Something wrong happened in Yunohost's code (most probably hook_exec)
-    except Exception:
-        import traceback
-
-        error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
-        logger.error(m18n.n("app_install_failed", app=app_id, error=error))
-        failure_message_with_debug_instructions = operation_logger.error(error)
     finally:
         # If success so far, validate that app didn't break important stuff
         if not install_failed:
@@ -1143,53 +1095,6 @@ def app_install(
     hook_callback("post_app_install", env=env_dict)
 
 
-def dump_app_log_extract_for_debugging(operation_logger):
-
-    with open(operation_logger.log_path, "r") as f:
-        lines = f.readlines()
-
-    filters = [
-        r"set [+-]x$",
-        r"set [+-]o xtrace$",
-        r"local \w+$",
-        r"local legacy_args=.*$",
-        r".*Helper used in legacy mode.*",
-        r"args_array=.*$",
-        r"local -A args_array$",
-        r"ynh_handle_getopts_args",
-        r"ynh_script_progression",
-    ]
-
-    filters = [re.compile(f_) for f_ in filters]
-
-    lines_to_display = []
-    for line in lines:
-
-        if ": " not in line.strip():
-            continue
-
-        # A line typically looks like
-        # 2019-10-19 16:10:27,611: DEBUG - + mysql -u piwigo --password=********** -B piwigo
-        # And we just want the part starting by "DEBUG - "
-        line = line.strip().split(": ", 1)[1]
-
-        if any(filter_.search(line) for filter_ in filters):
-            continue
-
-        lines_to_display.append(line)
-
-        if line.endswith("+ ynh_exit_properly") or " + ynh_die " in line:
-            break
-        elif len(lines_to_display) > 20:
-            lines_to_display.pop(0)
-
-    logger.warning(
-        "Here's an extract of the logs before the crash. It might help debugging the error:"
-    )
-    for line in lines_to_display:
-        logger.info(line)
-
-
 @is_unit_operation()
 def app_remove(operation_logger, app, purge=False):
     """
@@ -1760,171 +1665,100 @@ def app_action_run(operation_logger, app, action, args=None):
     return logger.success("Action successed!")
 
 
-# Config panel todo list:
+def app_config_get(app, key="", full=False, export=False):
-# * docstrings
+    """
-# * merge translations on the json once the workflow is in place
+    Display an app configuration in classic, full or export mode
+    """
+    if full and export:
+        raise YunohostValidationError("You can't use --full and --export together.", raw_msg=True)
+
+    if full:
+        mode = "full"
+    elif export:
+        mode = "export"
+    else:
+        mode = "classic"
+
+    config_ = AppConfigPanel(app)
+    return config_.get(key, mode)
+
+
 @is_unit_operation()
-def app_config_show_panel(operation_logger, app):
+def app_config_set(
-    logger.warning(m18n.n("experimental_feature"))
+    operation_logger, app, key=None, value=None, args=None, args_file=None
+):
+    """
+    Apply a new app configuration
+    """
 
-    from yunohost.hook import hook_exec
+    config_ = AppConfigPanel(app)
 
-    # this will take care of checking if the app is installed
+    Question.operation_logger = operation_logger
-    app_info_dict = app_info(app)
 
-    operation_logger.start()
+    return config_.set(key, value, args, args_file, operation_logger=operation_logger)
-    config_panel = _get_app_config_panel(app)
-    config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config")
 
-    app_id, app_instance_nb = _parse_app_instance_name(app)
 
-    if not config_panel or not os.path.exists(config_script):
+class AppConfigPanel(ConfigPanel):
-        return {
+    def __init__(self, app):
-            "app_id": app_id,
-            "app": app,
-            "app_name": app_info_dict["name"],
-            "config_panel": [],
-        }
 
-    env = {
+        # Check app is installed
-        "YNH_APP_ID": app_id,
+        _assert_is_installed(app)
-        "YNH_APP_INSTANCE_NAME": app,
-        "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
-    }
 
-    # FIXME: this should probably be ran in a tmp workdir...
+        self.app = app
-    return_code, parsed_values = hook_exec(
+        config_path = os.path.join(APPS_SETTING_PATH, app, "config_panel.toml")
-        config_script, args=["show"], env=env, return_format="plain_dict"
+        super().__init__(config_path=config_path)
-    )
 
-    if return_code != 0:
+    def _load_current_values(self):
-        raise Exception(
+        self.values = self._call_config_script("show")
-            "script/config show return value code: %s (considered as an error)",
-            return_code,
-        )
 
-    logger.debug("Generating global variables:")
+    def _apply(self):
-    for tab in config_panel.get("panel", []):
+        env = {key: str(value) for key, value in self.new_values.items()}
-        tab_id = tab["id"]  # this makes things easier to debug on crash
+        return_content = self._call_config_script("apply", env=env)
-        for section in tab.get("sections", []):
+
-            section_id = section["id"]
+        # If the script returned validation error
-            for option in section.get("options", []):
+        # raise a ValidationError exception using
-                option_name = option["name"]
+        # the first key
-                generated_name = (
+        if return_content:
-                    "YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)
+            for key, message in return_content.get("validation_errors").items():
-                ).upper()
+                raise YunohostValidationError(
-                option["name"] = generated_name
+                    "app_argument_invalid",
-                logger.debug(
+                    name=key,
-                    " * '%s'.'%s'.'%s' -> %s",
+                    error=message,
-                    tab.get("name"),
-                    section.get("name"),
-                    option.get("name"),
-                    generated_name,
                 )
 
-                if generated_name in parsed_values:
+    def _call_config_script(self, action, env={}):
-                    # code is not adapted for that so we have to mock expected format :/
+        from yunohost.hook import hook_exec
-                    if option.get("type") == "boolean":
-                        if parsed_values[generated_name].lower() in ("true", "1", "y"):
-                            option["default"] = parsed_values[generated_name]
-                        else:
-                            del option["default"]
-                    else:
-                        option["default"] = parsed_values[generated_name]
 
-                    args_dict = _parse_args_in_yunohost_format(
+        # Add default config script if needed
-                        {option["name"]: parsed_values[generated_name]}, [option]
+        config_script = os.path.join(APPS_SETTING_PATH, self.app, "scripts", "config")
-                    )
+        if not os.path.exists(config_script):
-                    option["default"] = args_dict[option["name"]][0]
+            logger.debug("Adding a default config script")
-                else:
+            default_script = """#!/bin/bash
-                    logger.debug(
+source /usr/share/yunohost/helpers
-                        "Variable '%s' is not declared by config script, using default",
+ynh_abort_if_errors
-                        generated_name,
+final_path=$(ynh_app_setting_get $app final_path)
-                    )
+ynh_app_config_run $1
-                    # do nothing, we'll use the default if present
+"""
+            write_to_file(config_script, default_script)
 
-    return {
+        # Call config script to extract current values
-        "app_id": app_id,
+        logger.debug(f"Calling '{action}' action from config script")
-        "app": app,
+        app_id, app_instance_nb = _parse_app_instance_name(self.app)
-        "app_name": app_info_dict["name"],
+        env.update(
-        "config_panel": config_panel,
+            {
-        "logs": operation_logger.success(),
+                "app_id": app_id,
-    }
+                "app": self.app,
-
+                "app_instance_nb": str(app_instance_nb),
-
+            }
-@is_unit_operation()
-def app_config_apply(operation_logger, app, args):
-    logger.warning(m18n.n("experimental_feature"))
-
-    from yunohost.hook import hook_exec
-
-    installed = _is_installed(app)
-    if not installed:
-        raise YunohostValidationError(
-            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
         )
 
-    config_panel = _get_app_config_panel(app)
+        ret, values = hook_exec(config_script, args=[action], env=env)
-    config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config")
+        if ret != 0:
-
+            if action == "show":
-    if not config_panel or not os.path.exists(config_script):
+                raise YunohostError("app_config_unable_to_read")
-        # XXX real exception
+            else:
-        raise Exception("Not config-panel.json nor scripts/config")
+                raise YunohostError("app_config_unable_to_apply")
-
+        return values
-    operation_logger.start()
-    app_id, app_instance_nb = _parse_app_instance_name(app)
-    env = {
-        "YNH_APP_ID": app_id,
-        "YNH_APP_INSTANCE_NAME": app,
-        "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
-    }
-    args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
-
-    for tab in config_panel.get("panel", []):
-        tab_id = tab["id"]  # this makes things easier to debug on crash
-        for section in tab.get("sections", []):
-            section_id = section["id"]
-            for option in section.get("options", []):
-                option_name = option["name"]
-                generated_name = (
-                    "YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)
-                ).upper()
-
-                if generated_name in args:
-                    logger.debug(
-                        "include into env %s=%s", generated_name, args[generated_name]
-                    )
-                    env[generated_name] = args[generated_name]
-                else:
-                    logger.debug("no value for key id %s", generated_name)
-
-    # for debug purpose
-    for key in args:
-        if key not in env:
-            logger.warning(
-                "Ignore key '%s' from arguments because it is not in the config", key
-            )
-
-    # FIXME: this should probably be ran in a tmp workdir...
-    return_code = hook_exec(
-        config_script,
-        args=["apply"],
-        env=env,
-    )[0]
-
-    if return_code != 0:
-        msg = (
-            "'script/config apply' return value code: %s (considered as an error)"
-            % return_code
-        )
-        operation_logger.error(msg)
-        raise Exception(msg)
-
-    logger.success("Config updated as expected")
-    return {
-        "app": app,
-        "logs": operation_logger.success(),
-    }
 
 
 def _get_all_installed_apps_id():
@@ -2028,145 +1862,6 @@ def _get_app_actions(app_id):
     return None
 
 
-def _get_app_config_panel(app_id):
-    "Get app config panel stored in json or in toml"
-    config_panel_toml_path = os.path.join(
-        APPS_SETTING_PATH, app_id, "config_panel.toml"
-    )
-    config_panel_json_path = os.path.join(
-        APPS_SETTING_PATH, app_id, "config_panel.json"
-    )
-
-    # sample data to get an idea of what is going on
-    # this toml extract:
-    #
-    # version = "0.1"
-    # name = "Unattended-upgrades configuration panel"
-    #
-    # [main]
-    # name = "Unattended-upgrades configuration"
-    #
-    #     [main.unattended_configuration]
-    #     name = "50unattended-upgrades configuration file"
-    #
-    #         [main.unattended_configuration.upgrade_level]
-    #         name = "Choose the sources of packages to automatically upgrade."
-    #         default = "Security only"
-    #         type = "text"
-    #         help = "We can't use a choices field for now. In the meantime please choose between one of this values:<br>Security only, Security and updates."
-    #         # choices = ["Security only", "Security and updates"]
-
-    #         [main.unattended_configuration.ynh_update]
-    #         name = "Would you like to update YunoHost packages automatically ?"
-    #         type = "bool"
-    #         default = true
-    #
-    # will be parsed into this:
-    #
-    # OrderedDict([(u'version', u'0.1'),
-    #              (u'name', u'Unattended-upgrades configuration panel'),
-    #              (u'main',
-    #               OrderedDict([(u'name', u'Unattended-upgrades configuration'),
-    #                            (u'unattended_configuration',
-    #                             OrderedDict([(u'name',
-    #                                           u'50unattended-upgrades configuration file'),
-    #                                          (u'upgrade_level',
-    #                                           OrderedDict([(u'name',
-    #                                                         u'Choose the sources of packages to automatically upgrade.'),
-    #                                                        (u'default',
-    #                                                         u'Security only'),
-    #                                                        (u'type', u'text'),
-    #                                                        (u'help',
-    #                                                         u"We can't use a choices field for now. In the meantime please choose between one of this values:<br>Security only, Security and updates.")])),
-    #                                          (u'ynh_update',
-    #                                           OrderedDict([(u'name',
-    #                                                         u'Would you like to update YunoHost packages automatically ?'),
-    #                                                        (u'type', u'bool'),
-    #                                                        (u'default', True)])),
-    #
-    # and needs to be converted into this:
-    #
-    # {u'name': u'Unattended-upgrades configuration panel',
-    #  u'panel': [{u'id': u'main',
-    #    u'name': u'Unattended-upgrades configuration',
-    #    u'sections': [{u'id': u'unattended_configuration',
-    #      u'name': u'50unattended-upgrades configuration file',
-    #      u'options': [{u'//': u'"choices" : ["Security only", "Security and updates"]',
-    #        u'default': u'Security only',
-    #        u'help': u"We can't use a choices field for now. In the meantime please choose between one of this values:<br>Security only, Security and updates.",
-    #        u'id': u'upgrade_level',
-    #        u'name': u'Choose the sources of packages to automatically upgrade.',
-    #        u'type': u'text'},
-    #       {u'default': True,
-    #        u'id': u'ynh_update',
-    #        u'name': u'Would you like to update YunoHost packages automatically ?',
-    #        u'type': u'bool'},
-
-    if os.path.exists(config_panel_toml_path):
-        toml_config_panel = toml.load(
-            open(config_panel_toml_path, "r"), _dict=OrderedDict
-        )
-
-        # transform toml format into json format
-        config_panel = {
-            "name": toml_config_panel["name"],
-            "version": toml_config_panel["version"],
-            "panel": [],
-        }
-
-        panels = [
-            key_value
-            for key_value in toml_config_panel.items()
-            if key_value[0] not in ("name", "version")
-            and isinstance(key_value[1], OrderedDict)
-        ]
-
-        for key, value in panels:
-            panel = {
-                "id": key,
-                "name": value["name"],
-                "sections": [],
-            }
-
-            sections = [
-                k_v1
-                for k_v1 in value.items()
-                if k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict)
-            ]
-
-            for section_key, section_value in sections:
-                section = {
-                    "id": section_key,
-                    "name": section_value["name"],
-                    "options": [],
-                }
-
-                options = [
-                    k_v
-                    for k_v in section_value.items()
-                    if k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict)
-                ]
-
-                for option_key, option_value in options:
-                    option = dict(option_value)
-                    option["name"] = option_key
-                    option["ask"] = {"en": option["ask"]}
-                    if "help" in option:
-                        option["help"] = {"en": option["help"]}
-                    section["options"].append(option)
-
-                panel["sections"].append(section)
-
-            config_panel["panel"].append(panel)
-
-        return config_panel
-
-    elif os.path.exists(config_panel_json_path):
-        return json.load(open(config_panel_json_path))
-
-    return None
-
-
 def _get_app_settings(app_id):
     """
     Get settings of an installed app
@@ -2607,34 +2302,17 @@ def _is_installed(app):
     return os.path.isdir(APPS_SETTING_PATH + app)
 
 
+def _assert_is_installed(app):
+    if not _is_installed(app):
+        raise YunohostValidationError(
+            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
+        )
+
+
 def _installed_apps():
     return os.listdir(APPS_SETTING_PATH)
 
 
-def _value_for_locale(values):
-    """
-    Return proper value for current locale
-
-    Keyword arguments:
-        values -- A dict of values associated to their locale
-
-    Returns:
-        An utf-8 encoded string
-
-    """
-    if not isinstance(values, dict):
-        return values
-
-    for lang in [m18n.locale, m18n.default_locale]:
-        try:
-            return values[lang]
-        except KeyError:
-            continue
-
-    # Fallback to first value
-    return list(values.values())[0]
-
-
 def _check_manifest_requirements(manifest, app_instance_name):
     """Check if required packages are met from the manifest"""
 
@@ -2681,7 +2359,7 @@ def _parse_args_from_manifest(manifest, action, args={}):
         return OrderedDict()
 
     action_args = manifest["arguments"][action]
-    return _parse_args_in_yunohost_format(args, action_args)
+    return parse_args_in_yunohost_format(args, action_args)
 
 
 def _parse_args_for_action(action, args={}):
@@ -2705,298 +2383,7 @@ def _parse_args_for_action(action, args={}):
 
     action_args = action["arguments"]
 
-    return _parse_args_in_yunohost_format(args, action_args)
+    return parse_args_in_yunohost_format(args, action_args)
-
-
-class Question:
-    "empty class to store questions information"
-
-
-class YunoHostArgumentFormatParser(object):
-    hide_user_input_in_prompt = False
-
-    def parse_question(self, question, user_answers):
-        parsed_question = Question()
-
-        parsed_question.name = question["name"]
-        parsed_question.default = question.get("default", None)
-        parsed_question.choices = question.get("choices", [])
-        parsed_question.optional = question.get("optional", False)
-        parsed_question.ask = question.get("ask")
-        parsed_question.value = user_answers.get(parsed_question.name)
-
-        if parsed_question.ask is None:
-            parsed_question.ask = "Enter value for '%s':" % parsed_question.name
-
-        # Empty value is parsed as empty string
-        if parsed_question.default == "":
-            parsed_question.default = None
-
-        return parsed_question
-
-    def parse(self, question, user_answers):
-        question = self.parse_question(question, user_answers)
-
-        if question.value is None:
-            text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
-                question
-            )
-
-            try:
-                question.value = Moulinette.prompt(
-                    text_for_user_input_in_cli, self.hide_user_input_in_prompt
-                )
-            except NotImplementedError:
-                question.value = None
-
-        # we don't have an answer, check optional and default_value
-        if question.value is None or question.value == "":
-            if not question.optional and question.default is None:
-                raise YunohostValidationError(
-                    "app_argument_required", name=question.name
-                )
-            else:
-                question.value = (
-                    getattr(self, "default_value", None)
-                    if question.default is None
-                    else question.default
-                )
-
-        # we have an answer, do some post checks
-        if question.value is not None:
-            if question.choices and question.value not in question.choices:
-                self._raise_invalid_answer(question)
-
-        # this is done to enforce a certain formating like for boolean
-        # by default it doesn't do anything
-        question.value = self._post_parse_value(question)
-
-        return (question.value, self.argument_type)
-
-    def _raise_invalid_answer(self, question):
-        raise YunohostValidationError(
-            "app_argument_choice_invalid",
-            name=question.name,
-            choices=", ".join(question.choices),
-        )
-
-    def _format_text_for_user_input_in_cli(self, question):
-        text_for_user_input_in_cli = _value_for_locale(question.ask)
-
-        if question.choices:
-            text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices))
-
-        if question.default is not None:
-            text_for_user_input_in_cli += " (default: {0})".format(question.default)
-
-        return text_for_user_input_in_cli
-
-    def _post_parse_value(self, question):
-        return question.value
-
-
-class StringArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "string"
-    default_value = ""
-
-
-class PasswordArgumentParser(YunoHostArgumentFormatParser):
-    hide_user_input_in_prompt = True
-    argument_type = "password"
-    default_value = ""
-    forbidden_chars = "{}"
-
-    def parse_question(self, question, user_answers):
-        question = super(PasswordArgumentParser, self).parse_question(
-            question, user_answers
-        )
-
-        if question.default is not None:
-            raise YunohostValidationError(
-                "app_argument_password_no_default", name=question.name
-            )
-
-        return question
-
-    def _post_parse_value(self, question):
-        if any(char in question.value for char in self.forbidden_chars):
-            raise YunohostValidationError(
-                "pattern_password_app", forbidden_chars=self.forbidden_chars
-            )
-
-        # If it's an optional argument the value should be empty or strong enough
-        if not question.optional or question.value:
-            from yunohost.utils.password import assert_password_is_strong_enough
-
-            assert_password_is_strong_enough("user", question.value)
-
-        return super(PasswordArgumentParser, self)._post_parse_value(question)
-
-
-class PathArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "path"
-    default_value = ""
-
-
-class BooleanArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "boolean"
-    default_value = False
-
-    def parse_question(self, question, user_answers):
-        question = super(BooleanArgumentParser, self).parse_question(
-            question, user_answers
-        )
-
-        if question.default is None:
-            question.default = False
-
-        return question
-
-    def _format_text_for_user_input_in_cli(self, question):
-        text_for_user_input_in_cli = _value_for_locale(question.ask)
-
-        text_for_user_input_in_cli += " [yes | no]"
-
-        if question.default is not None:
-            formatted_default = "yes" if question.default else "no"
-            text_for_user_input_in_cli += " (default: {0})".format(formatted_default)
-
-        return text_for_user_input_in_cli
-
-    def _post_parse_value(self, question):
-        if isinstance(question.value, bool):
-            return 1 if question.value else 0
-
-        if str(question.value).lower() in ["1", "yes", "y", "true"]:
-            return 1
-
-        if str(question.value).lower() in ["0", "no", "n", "false"]:
-            return 0
-
-        raise YunohostValidationError(
-            "app_argument_choice_invalid",
-            name=question.name,
-            choices="yes, no, y, n, 1, 0",
-        )
-
-
-class DomainArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "domain"
-
-    def parse_question(self, question, user_answers):
-        from yunohost.domain import domain_list, _get_maindomain
-
-        question = super(DomainArgumentParser, self).parse_question(
-            question, user_answers
-        )
-
-        if question.default is None:
-            question.default = _get_maindomain()
-
-        question.choices = domain_list()["domains"]
-
-        return question
-
-    def _raise_invalid_answer(self, question):
-        raise YunohostValidationError(
-            "app_argument_invalid", name=question.name, error=m18n.n("domain_unknown")
-        )
-
-
-class UserArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "user"
-
-    def parse_question(self, question, user_answers):
-        from yunohost.user import user_list, user_info
-        from yunohost.domain import _get_maindomain
-
-        question = super(UserArgumentParser, self).parse_question(
-            question, user_answers
-        )
-        question.choices = user_list()["users"]
-        if question.default is None:
-            root_mail = "root@%s" % _get_maindomain()
-            for user in question.choices.keys():
-                if root_mail in user_info(user).get("mail-aliases", []):
-                    question.default = user
-                    break
-
-        return question
-
-    def _raise_invalid_answer(self, question):
-        raise YunohostValidationError(
-            "app_argument_invalid",
-            name=question.name,
-            error=m18n.n("user_unknown", user=question.value),
-        )
-
-
-class NumberArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "number"
-    default_value = ""
-
-    def parse_question(self, question, user_answers):
-        question = super(NumberArgumentParser, self).parse_question(
-            question, user_answers
-        )
-
-        if question.default is None:
-            question.default = 0
-
-        return question
-
-    def _post_parse_value(self, question):
-        if isinstance(question.value, int):
-            return super(NumberArgumentParser, self)._post_parse_value(question)
-
-        if isinstance(question.value, str) and question.value.isdigit():
-            return int(question.value)
-
-        raise YunohostValidationError(
-            "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
-        )
-
-
-class DisplayTextArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "display_text"
-
-    def parse(self, question, user_answers):
-        print(question["ask"])
-
-
-ARGUMENTS_TYPE_PARSERS = {
-    "string": StringArgumentParser,
-    "password": PasswordArgumentParser,
-    "path": PathArgumentParser,
-    "boolean": BooleanArgumentParser,
-    "domain": DomainArgumentParser,
-    "user": UserArgumentParser,
-    "number": NumberArgumentParser,
-    "display_text": DisplayTextArgumentParser,
-}
-
-
-def _parse_args_in_yunohost_format(user_answers, argument_questions):
-    """Parse arguments store in either manifest.json or actions.json or from a
-    config panel against the user answers when they are present.
-
-    Keyword arguments:
-        user_answers -- a dictionnary of arguments from the user (generally
-                        empty in CLI, filed from the admin interface)
-        argument_questions -- the arguments description store in yunohost
-                              format from actions.json/toml, manifest.json/toml
-                              or config_panel.json/toml
-    """
-    parsed_answers_dict = OrderedDict()
-
-    for question in argument_questions:
-        parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]()
-
-        answer = parser.parse(question=question, user_answers=user_answers)
-        if answer is not None:
-            parsed_answers_dict[question["name"]] = answer
-
-    return parsed_answers_dict
 
 
 def _validate_and_normalize_webpath(args_dict, app_folder):
@@ -3420,6 +2807,8 @@ def unstable_apps():
 
 def _assert_system_is_sane_for_app(manifest, when):
 
+    from yunohost.service import service_status
+
     logger.debug("Checking that required services are up and running...")
 
     services = manifest.get("services", [])

src/yunohost/backup.py

@@ -48,7 +48,6 @@ from yunohost.app import (
     app_info,
     _is_installed,
     _make_environment_for_app_script,
-    dump_app_log_extract_for_debugging,
     _patch_legacy_helpers,
     _patch_legacy_php_versions,
     _patch_legacy_php_versions_in_settings,
@@ -60,6 +59,7 @@ from yunohost.hook import (
     hook_info,
     hook_callback,
     hook_exec,
+    hook_exec_with_script_debug_if_failure,
     CUSTOM_HOOK_FOLDER,
 )
 from yunohost.tools import (
@@ -1496,37 +1496,14 @@ class RestoreManager:
         # Execute the app install script
         restore_failed = True
         try:
-            restore_retcode = hook_exec(
+            restore_failed, failure_message_with_debug_instructions = hook_exec_with_script_debug_if_failure(
                 restore_script,
                 chdir=app_backup_in_archive,
                 env=env_dict,
-            )[0]
+                operation_logger=operation_logger,
-            # "Common" app restore failure : the script failed and returned exit code != 0
+                error_message_if_script_failed=m18n.n("app_restore_script_failed"),
-            restore_failed = True if restore_retcode != 0 else False
+                error_message_if_failed=lambda e: m18n.n("app_restore_failed", app=app_instance_name, error=e)
-            if restore_failed:
-                error = m18n.n("app_restore_script_failed")
-                logger.error(
-                    m18n.n("app_restore_failed", app=app_instance_name, error=error)
-                )
-                failure_message_with_debug_instructions = operation_logger.error(error)
-                if Moulinette.interface.type != "api":
-                    dump_app_log_extract_for_debugging(operation_logger)
-        # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception
-        except (KeyboardInterrupt, EOFError):
-            error = m18n.n("operation_interrupted")
-            logger.error(
-                m18n.n("app_restore_failed", app=app_instance_name, error=error)
             )
-            failure_message_with_debug_instructions = operation_logger.error(error)
-        # Something wrong happened in Yunohost's code (most probably hook_exec)
-        except Exception:
-            import traceback
-
-            error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
-            logger.error(
-                m18n.n("app_restore_failed", app=app_instance_name, error=error)
-            )
-            failure_message_with_debug_instructions = operation_logger.error(error)
         finally:
             # Cleaning temporary scripts directory
             shutil.rmtree(tmp_workdir_for_app, ignore_errors=True)

src/yunohost/hook.py

@@ -34,7 +34,7 @@ from importlib import import_module
 from moulinette import m18n, Moulinette
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from moulinette.utils import log
-from moulinette.utils.filesystem import read_json
+from moulinette.utils.filesystem import read_yaml
 
 HOOK_FOLDER = "/usr/share/yunohost/hooks/"
 CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/"
@@ -326,7 +326,7 @@ def hook_exec(
     chdir=None,
     env=None,
     user="root",
-    return_format="json",
+    return_format="yaml",
 ):
     """
     Execute hook from a file with arguments
@@ -447,10 +447,10 @@ def _hook_exec_bash(path, args, chdir, env, user, return_format, loggers):
             raw_content = f.read()
         returncontent = {}
 
-        if return_format == "json":
+        if return_format == "yaml":
             if raw_content != "":
                 try:
-                    returncontent = read_json(stdreturn)
+                    returncontent = read_yaml(stdreturn)
                 except Exception as e:
                     raise YunohostError(
                         "hook_json_return_error",
@@ -498,6 +498,40 @@ def _hook_exec_python(path, args, env, loggers):
     return ret
 
 
+def hook_exec_with_script_debug_if_failure(*args, **kwargs):
+
+    operation_logger = kwargs.pop("operation_logger")
+    error_message_if_failed = kwargs.pop("error_message_if_failed")
+    error_message_if_script_failed = kwargs.pop("error_message_if_script_failed")
+
+    failed = True
+    failure_message_with_debug_instructions = None
+    try:
+        retcode, retpayload = hook_exec(*args, **kwargs)
+        failed = True if retcode != 0 else False
+        if failed:
+            error = error_message_if_script_failed
+            logger.error(error_message_if_failed(error))
+            failure_message_with_debug_instructions = operation_logger.error(error)
+            if Moulinette.interface.type != "api":
+                operation_logger.dump_script_log_extract_for_debugging()
+    # Script got manually interrupted ...
+    # N.B. : KeyboardInterrupt does not inherit from Exception
+    except (KeyboardInterrupt, EOFError):
+        error = m18n.n("operation_interrupted")
+        logger.error(error_message_if_failed(error))
+        failure_message_with_debug_instructions = operation_logger.error(error)
+    # Something wrong happened in Yunohost's code (most probably hook_exec)
+    except Exception:
+        import traceback
+
+        error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
+        logger.error(error_message_if_failed(error))
+        failure_message_with_debug_instructions = operation_logger.error(error)
+
+    return failed, failure_message_with_debug_instructions
+
+
 def _extract_filename_parts(filename):
     """Extract hook parts from filename"""
     if "-" in filename:

src/yunohost/log.py

@@ -707,6 +707,52 @@ class OperationLogger(object):
         else:
             self.error(m18n.n("log_operation_unit_unclosed_properly"))
 
+    def dump_script_log_extract_for_debugging(self):
+
+        with open(self.log_path, "r") as f:
+            lines = f.readlines()
+
+        filters = [
+            r"set [+-]x$",
+            r"set [+-]o xtrace$",
+            r"local \w+$",
+            r"local legacy_args=.*$",
+            r".*Helper used in legacy mode.*",
+            r"args_array=.*$",
+            r"local -A args_array$",
+            r"ynh_handle_getopts_args",
+            r"ynh_script_progression",
+        ]
+
+        filters = [re.compile(f_) for f_ in filters]
+
+        lines_to_display = []
+        for line in lines:
+
+            if ": " not in line.strip():
+                continue
+
+            # A line typically looks like
+            # 2019-10-19 16:10:27,611: DEBUG - + mysql -u piwigo --password=********** -B piwigo
+            # And we just want the part starting by "DEBUG - "
+            line = line.strip().split(": ", 1)[1]
+
+            if any(filter_.search(line) for filter_ in filters):
+                continue
+
+            lines_to_display.append(line)
+
+            if line.endswith("+ ynh_exit_properly") or " + ynh_die " in line:
+                break
+            elif len(lines_to_display) > 20:
+                lines_to_display.pop(0)
+
+        logger.warning(
+            "Here's an extract of the logs before the crash. It might help debugging the error:"
+        )
+        for line in lines_to_display:
+            logger.info(line)
+
 
 def _get_datetime_from_name(name):
 

src/yunohost/service.py

@@ -256,7 +256,7 @@ def service_restart(names):
                 )
 
 
-def service_reload_or_restart(names):
+def service_reload_or_restart(names, test_conf=True):
     """
     Reload one or more services if they support it. If not, restart them instead. If the services are not running yet, they will be started.
 
@@ -266,7 +266,32 @@ def service_reload_or_restart(names):
     """
     if isinstance(names, str):
         names = [names]
+
+    services = _get_services()
+
     for name in names:
+
+        logger.debug(f"Reloading service {name}")
+
+        test_conf_cmd = services.get(name, {}).get("test_conf")
+        if test_conf and test_conf_cmd:
+
+            p = subprocess.Popen(
+                test_conf_cmd,
+                shell=True,
+                executable="/bin/bash",
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            )
+
+            out, _ = p.communicate()
+            if p.returncode != 0:
+                errors = out.decode().strip().split("\n")
+                logger.error(
+                    m18n.n("service_not_reloading_because_conf_broken", name=name, errors=errors)
+                )
+                continue
+
         if _run_service_command("reload-or-restart", name):
             logger.success(m18n.n("service_reloaded_or_restarted", service=name))
         else:

src/yunohost/tests/conftest.py

@@ -84,9 +84,12 @@ def pytest_cmdline_main(config):
 
     class DummyInterface:
 
-        type = "test"
+        type = "cli"
 
-        def prompt(*args, **kwargs):
+        def prompt(self, *args, **kwargs):
             raise NotImplementedError
 
+        def display(self, message, *args, **kwargs):
+            print(message)
+
     Moulinette._interface = DummyInterface()

src/yunohost/tests/test_app_config.py

@@ -0,0 +1,200 @@
+import glob
+import os
+import shutil
+import pytest
+
+from .conftest import get_test_apps_dir
+
+from moulinette.utils.filesystem import read_file
+
+from yunohost.domain import _get_maindomain
+from yunohost.app import (
+    app_setting,
+    app_install,
+    app_remove,
+    _is_installed,
+    app_config_get,
+    app_config_set,
+    app_ssowatconf,
+)
+
+from yunohost.utils.error import YunohostError, YunohostValidationError
+
+
+def setup_function(function):
+
+    clean()
+
+
+def teardown_function(function):
+
+    clean()
+
+
+def clean():
+
+    # Make sure we have a ssowat
+    os.system("mkdir -p /etc/ssowat/")
+    app_ssowatconf()
+
+    test_apps = ["config_app", "legacy_app"]
+
+    for test_app in test_apps:
+
+        if _is_installed(test_app):
+            app_remove(test_app)
+
+        for filepath in glob.glob("/etc/nginx/conf.d/*.d/*%s*" % test_app):
+            os.remove(filepath)
+        for folderpath in glob.glob("/etc/yunohost/apps/*%s*" % test_app):
+            shutil.rmtree(folderpath, ignore_errors=True)
+        for folderpath in glob.glob("/var/www/*%s*" % test_app):
+            shutil.rmtree(folderpath, ignore_errors=True)
+
+        os.system("bash -c \"mysql -B 2>/dev/null <<< 'DROP DATABASE %s' \"" % test_app)
+        os.system(
+            "bash -c \"mysql -B 2>/dev/null <<< 'DROP USER %s@localhost'\"" % test_app
+        )
+
+    # Reset failed quota for service to avoid running into start-limit rate ?
+    os.system("systemctl reset-failed nginx")
+    os.system("systemctl start nginx")
+
+
+@pytest.fixture()
+def legacy_app(request):
+
+    main_domain = _get_maindomain()
+
+    app_install(
+        os.path.join(get_test_apps_dir(), "legacy_app_ynh"),
+        args="domain=%s&path=%s&is_public=%s" % (main_domain, "/", 1),
+        force=True,
+    )
+
+    def remove_app():
+        app_remove("legacy_app")
+
+    request.addfinalizer(remove_app)
+
+    return "legacy_app"
+
+
+
+@pytest.fixture()
+def config_app(request):
+
+    app_install(
+        os.path.join(get_test_apps_dir(), "config_app_ynh"),
+        args="",
+        force=True,
+    )
+
+    def remove_app():
+        app_remove("config_app")
+
+    request.addfinalizer(remove_app)
+
+    return "config_app"
+
+
+def test_app_config_get(config_app):
+
+    assert isinstance(app_config_get(config_app), dict)
+    assert isinstance(app_config_get(config_app, full=True), dict)
+    assert isinstance(app_config_get(config_app, export=True), dict)
+    assert isinstance(app_config_get(config_app, "main"), dict)
+    assert isinstance(app_config_get(config_app, "main.components"), dict)
+    assert app_config_get(config_app, "main.components.boolean") == "0"
+
+
+def test_app_config_nopanel(legacy_app):
+
+    with pytest.raises(YunohostValidationError):
+        app_config_get(legacy_app)
+
+
+def test_app_config_get_nonexistentstuff(config_app):
+
+    with pytest.raises(YunohostValidationError):
+        app_config_get("nonexistent")
+
+    with pytest.raises(YunohostValidationError):
+        app_config_get(config_app, "nonexistent")
+
+    with pytest.raises(YunohostValidationError):
+        app_config_get(config_app, "main.nonexistent")
+
+    with pytest.raises(YunohostValidationError):
+        app_config_get(config_app, "main.components.nonexistent")
+
+    app_setting(config_app, "boolean", delete=True)
+    with pytest.raises(YunohostValidationError):
+        app_config_get(config_app, "main.components.boolean")
+
+
+def test_app_config_regular_setting(config_app):
+
+    assert app_config_get(config_app, "main.components.boolean") == "0"
+
+    app_config_set(config_app, "main.components.boolean", "no")
+
+    assert app_config_get(config_app, "main.components.boolean") == "0"
+    assert app_setting(config_app, "boolean") == "0"
+
+    app_config_set(config_app, "main.components.boolean", "yes")
+
+    assert app_config_get(config_app, "main.components.boolean") == "1"
+    assert app_setting(config_app, "boolean") == "1"
+
+    with pytest.raises(YunohostValidationError):
+        app_config_set(config_app, "main.components.boolean", "pwet")
+
+
+def test_app_config_bind_on_file(config_app):
+
+    # c.f. conf/test.php in the config app
+    assert '$arg5= "Arg5 value";' in read_file("/var/www/config_app/test.php")
+    assert app_config_get(config_app, "bind.variable.arg5") == "Arg5 value"
+    assert app_setting(config_app, "arg5") is None
+
+    app_config_set(config_app, "bind.variable.arg5", "Foo Bar")
+
+    assert '$arg5= "Foo Bar";' in read_file("/var/www/config_app/test.php")
+    assert app_config_get(config_app, "bind.variable.arg5") == "Foo Bar"
+    assert app_setting(config_app, "arg5") == "Foo Bar"
+
+
+def test_app_config_custom_get(config_app):
+
+    assert app_setting(config_app, "arg9") is None
+    assert "Files in /var/www" in app_config_get(config_app, "bind.function.arg9")["ask"]["en"]
+    assert app_setting(config_app, "arg9") is None
+
+
+def test_app_config_custom_validator(config_app):
+
+    # c.f. the config script
+    # arg8 is a password that must be at least 8 chars
+    assert not os.path.exists("/var/www/config_app/password")
+    assert app_setting(config_app, "arg8") is None
+
+    with pytest.raises(YunohostValidationError):
+        app_config_set(config_app, "bind.function.arg8", "pZo6i7u91h")
+
+    assert not os.path.exists("/var/www/config_app/password")
+    assert app_setting(config_app, "arg8") is None
+
+
+def test_app_config_custom_set(config_app):
+
+    assert not os.path.exists("/var/www/config_app/password")
+    assert app_setting(config_app, "arg8") is None
+
+    app_config_set(config_app, "bind.function.arg8", "OneSuperStrongPassword")
+
+    assert os.path.exists("/var/www/config_app/password")
+    content = read_file("/var/www/config_app/password")
+    assert "OneSuperStrongPassword" not in content
+    assert content.startswith("$6$saltsalt$")
+    assert app_setting(config_app, "arg8") is None

src/yunohost/tests/test_backuprestore.py

@@ -469,7 +469,7 @@ def test_restore_app_script_failure_handling(monkeypatch, mocker):
             monkeypatch.undo()
             return (1, None)
 
-    monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec)
+    monkeypatch.setattr("yunohost.hook.hook_exec", custom_hook_exec)
 
     assert not _is_installed("wordpress")
 

src/yunohost/tests/test_service.py

@@ -9,6 +9,7 @@ from yunohost.service import (
     service_add,
     service_remove,
     service_log,
+    service_reload_or_restart,
 )
 
 
@@ -38,6 +39,10 @@ def clean():
 
     _save_services(services)
 
+    if os.path.exists("/etc/nginx/conf.d/broken.conf"):
+        os.remove("/etc/nginx/conf.d/broken.conf")
+        os.system("systemctl reload-or-restart nginx")
+
 
 def test_service_status_all():
 
@@ -118,3 +123,20 @@ def test_service_update_to_remove_properties():
     assert _get_services()["dummyservice"].get("test_status") == "false"
     service_add("dummyservice", description="dummy", test_status="")
     assert not _get_services()["dummyservice"].get("test_status")
+
+
+def test_service_conf_broken():
+
+    os.system("echo pwet > /etc/nginx/conf.d/broken.conf")
+
+    status = service_status("nginx")
+    assert status["status"] == "running"
+    assert status["configuration"] == "broken"
+    assert "broken.conf" in status["configuration-details"][0]
+
+    # Service reload-or-restart should check that the conf ain't valid
+    # before reload-or-restart, hence the service should still be running
+    service_reload_or_restart("nginx")
+    assert status["status"] == "running"
+
+    os.remove("/etc/nginx/conf.d/broken.conf")

src/yunohost/utils/config.py

@@ -0,0 +1,970 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2018 YUNOHOST.ORG
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+import os
+import re
+import urllib.parse
+import tempfile
+import shutil
+from collections import OrderedDict
+
+from moulinette.interfaces.cli import colorize
+from moulinette import Moulinette, m18n
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import (
+    write_to_file,
+    read_toml,
+    read_yaml,
+    write_to_yaml,
+    mkdir,
+)
+
+from yunohost.utils.i18n import _value_for_locale
+from yunohost.utils.error import YunohostError, YunohostValidationError
+
+logger = getActionLogger("yunohost.config")
+CONFIG_PANEL_VERSION_SUPPORTED = 1.0
+
+
+class ConfigPanel:
+    def __init__(self, config_path, save_path=None):
+        self.config_path = config_path
+        self.save_path = save_path
+        self.config = {}
+        self.values = {}
+        self.new_values = {}
+
+    def get(self, key="", mode="classic"):
+        self.filter_key = key or ""
+
+        # Read config panel toml
+        self._get_config_panel()
+
+        if not self.config:
+            raise YunohostValidationError("config_no_panel")
+
+        # Read or get values and hydrate the config
+        self._load_current_values()
+        self._hydrate()
+
+        # Format result in full mode
+        if mode == "full":
+            return self.config
+
+        # In 'classic' mode, we display the current value if key refer to an option
+        if self.filter_key.count(".") == 2 and mode == "classic":
+            option = self.filter_key.split(".")[-1]
+            return self.values.get(option, None)
+
+        # Format result in 'classic' or 'export' mode
+        logger.debug(f"Formating result in '{mode}' mode")
+        result = {}
+        for panel, section, option in self._iterate():
+            key = f"{panel['id']}.{section['id']}.{option['id']}"
+            if mode == "export":
+                result[option["id"]] = option.get("current_value")
+            else:
+                if "ask" in option:
+                    result[key] = {"ask": _value_for_locale(option["ask"])}
+                elif "i18n" in self.config:
+                    result[key] = {
+                        "ask": m18n.n(self.config["i18n"] + "_" + option["id"])
+                    }
+                if "current_value" in option:
+                    question_class = ARGUMENTS_TYPE_PARSERS[
+                        option.get("type", "string")
+                    ]
+                    result[key]["value"] = question_class.humanize(
+                        option["current_value"], option
+                    )
+
+        return result
+
+    def set(self, key=None, value=None, args=None, args_file=None, operation_logger=None):
+        self.filter_key = key or ""
+
+        # Read config panel toml
+        self._get_config_panel()
+
+        if not self.config:
+            raise YunohostValidationError("config_no_panel")
+
+        if (args is not None or args_file is not None) and value is not None:
+            raise YunohostValidationError("You should either provide a value, or a serie of args/args_file, but not both at the same time", raw_msg=True)
+
+        if self.filter_key.count(".") != 2 and value is not None:
+            raise YunohostValidationError("config_cant_set_value_on_section")
+
+        # Import and parse pre-answered options
+        logger.debug("Import and parse pre-answered options")
+        args = urllib.parse.parse_qs(args or "", keep_blank_values=True)
+        self.args = {key: ",".join(value_) for key, value_ in args.items()}
+
+        if args_file:
+            # Import YAML / JSON file but keep --args values
+            self.args = {**read_yaml(args_file), **self.args}
+
+        if value is not None:
+            self.args = {self.filter_key.split(".")[-1]: value}
+
+        # Read or get values and hydrate the config
+        self._load_current_values()
+        self._hydrate()
+        self._ask()
+
+        if operation_logger:
+            operation_logger.start()
+
+        try:
+            self._apply()
+        except YunohostError:
+            raise
+        # Script got manually interrupted ...
+        # N.B. : KeyboardInterrupt does not inherit from Exception
+        except (KeyboardInterrupt, EOFError):
+            error = m18n.n("operation_interrupted")
+            logger.error(m18n.n("config_apply_failed", error=error))
+            raise
+        # Something wrong happened in Yunohost's code (most probably hook_exec)
+        except Exception:
+            import traceback
+
+            error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
+            logger.error(m18n.n("config_apply_failed", error=error))
+            raise
+        finally:
+            # Delete files uploaded from API
+            FileQuestion.clean_upload_dirs()
+
+        self._reload_services()
+
+        logger.success("Config updated as expected")
+        operation_logger.success()
+
+    def _get_toml(self):
+        return read_toml(self.config_path)
+
+    def _get_config_panel(self):
+
+        # Split filter_key
+        filter_key = self.filter_key.split(".") if self.filter_key != "" else []
+        if len(filter_key) > 3:
+            raise YunohostError(f"The filter key {filter_key} has too many sub-levels, the max is 3.", raw_msg=True)
+
+        if not os.path.exists(self.config_path):
+            return None
+        toml_config_panel = self._get_toml()
+
+        # Check TOML config panel is in a supported version
+        if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED:
+            raise YunohostError(
+                "config_version_not_supported", version=toml_config_panel["version"]
+            )
+
+        # Transform toml format into internal format
+        format_description = {
+            "toml": {
+                "properties": ["version", "i18n"],
+                "default": {"version": 1.0},
+            },
+            "panels": {
+                "properties": ["name", "services", "actions", "help"],
+                "default": {
+                    "name": "",
+                    "services": [],
+                    "actions": {"apply": {"en": "Apply"}}
+                },
+            },
+            "sections": {
+                "properties": ["name", "services", "optional", "help", "visible"],
+                "default": {
+                    "name": "",
+                    "services": [],
+                    "optional": True,
+                }
+            },
+            "options": {
+                "properties": ["ask", "type", "bind", "help", "example", "default",
+                               "style", "icon", "placeholder", "visible",
+                               "optional", "choices", "yes", "no", "pattern",
+                               "limit", "min", "max", "step", "accept", "redact"],
+                "default": {}
+            }
+        }
+
+        def convert(toml_node, node_type):
+            """Convert TOML in internal format ('full' mode used by webadmin)
+            Here are some properties of 1.0 config panel in toml:
+            - node properties and node children are mixed,
+            - text are in english only
+            - some properties have default values
+            This function detects all children nodes and put them in a list
+            """
+            # Prefill the node default keys if needed
+            default = format_description[node_type]['default']
+            node = {key: toml_node.get(key, value) for key, value in default.items()}
+
+            properties = format_description[node_type]['properties']
+
+            # Define the filter_key part to use and the children type
+            i = list(format_description).index(node_type)
+            subnode_type = list(format_description)[i + 1] if node_type != "options" else None
+            search_key = filter_key[i] if len(filter_key) > i else False
+
+            for key, value in toml_node.items():
+                # Key/value are a child node
+                if (
+                    isinstance(value, OrderedDict)
+                    and key not in properties
+                    and subnode_type
+                ):
+                    # We exclude all nodes not referenced by the filter_key
+                    if search_key and key != search_key:
+                        continue
+                    subnode = convert(value, subnode_type)
+                    subnode["id"] = key
+                    if node_type == "sections":
+                        subnode["name"] = key  # legacy
+                        subnode.setdefault("optional", toml_node.get("optional", True))
+                    node.setdefault(subnode_type, []).append(subnode)
+                # Key/value are a property
+                else:
+                    if key not in properties:
+                        logger.warning(f"Unknown key '{key}' found in config toml")
+                    # Todo search all i18n keys
+                    node[key] = (
+                        value if key not in ["ask", "help", "name"] else {"en": value}
+                    )
+            return node
+
+        self.config = convert(toml_config_panel, "toml")
+
+        try:
+            self.config["panels"][0]["sections"][0]["options"][0]
+        except (KeyError, IndexError):
+            raise YunohostValidationError(
+                "config_unknown_filter_key", filter_key=self.filter_key
+            )
+
+        # List forbidden keywords from helpers and sections toml (to avoid conflict)
+        forbidden_keywords = ["old", "app", "changed", "file_hash", "binds", "types",
+                              "formats", "getter", "setter", "short_setting", "type",
+                              "bind", "nothing_changed", "changes_validated", "result",
+                              "max_progression"]
+        forbidden_keywords += format_description["sections"]
+
+        for _, _, option in self._iterate():
+            if option["id"] in forbidden_keywords:
+                raise YunohostError("config_forbidden_keyword", keyword=option["id"])
+        return self.config
+
+    def _hydrate(self):
+        # Hydrating config panel with current value
+        logger.debug("Hydrating config with current values")
+        for _, _, option in self._iterate():
+            if option["id"] not in self.values:
+                allowed_empty_types = ["alert", "display_text", "markdown", "file"]
+                if option["type"] in allowed_empty_types or option.get("bind") == "null":
+                    continue
+                else:
+                    raise YunohostError(f"Config panel question '{option['id']}' should be initialized with a value during install or upgrade.")
+            value = self.values[option["name"]]
+            # In general, the value is just a simple value.
+            # Sometimes it could be a dict used to overwrite the option itself
+            value = value if isinstance(value, dict) else {"current_value": value}
+            option.update(value)
+
+        return self.values
+
+    def _ask(self):
+        logger.debug("Ask unanswered question and prevalidate data")
+
+        def display_header(message):
+            """CLI panel/section header display"""
+            if Moulinette.interface.type == "cli" and self.filter_key.count(".") < 2:
+                Moulinette.display(colorize(message, "purple"))
+
+        for panel, section, obj in self._iterate(["panel", "section"]):
+            if panel == obj:
+                name = _value_for_locale(panel["name"])
+                display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
+                continue
+            name = _value_for_locale(section["name"])
+            display_header(f"\n# {name}")
+
+            # Check and ask unanswered questions
+            self.new_values.update(
+                parse_args_in_yunohost_format(self.args, section["options"])
+            )
+        self.new_values = {
+            key: value[0]
+            for key, value in self.new_values.items()
+            if not value[0] is None
+        }
+        self.errors = None
+
+    def _get_default_values(self):
+        return {
+            option["id"]: option["default"]
+            for _, _, option in self._iterate()
+            if "default" in option
+        }
+
+    def _load_current_values(self):
+        """
+        Retrieve entries in YAML file
+        And set default values if needed
+        """
+
+        # Retrieve entries in the YAML
+        on_disk_settings = {}
+        if os.path.exists(self.save_path) and os.path.isfile(self.save_path):
+            on_disk_settings = read_yaml(self.save_path) or {}
+
+        # Inject defaults if needed (using the magic .update() ;))
+        self.values = self._get_default_values()
+        self.values.update(on_disk_settings)
+
+    def _apply(self):
+        logger.info("Saving the new configuration...")
+        dir_path = os.path.dirname(os.path.realpath(self.save_path))
+        if not os.path.exists(dir_path):
+            mkdir(dir_path, mode=0o700)
+
+        values_to_save = {**self.values, **self.new_values}
+        if self.save_mode == "diff":
+            defaults = self._get_default_values()
+            values_to_save = {
+                k: v for k, v in values_to_save.items() if defaults.get(k) != v
+            }
+
+        # Save the settings to the .yaml file
+        write_to_yaml(self.save_path, self.new_values)
+
+    def _reload_services(self):
+
+        from yunohost.service import service_reload_or_restart
+
+        services_to_reload = set()
+        for panel, section, obj in self._iterate(["panel", "section", "option"]):
+            services_to_reload |= set(obj.get("services", []))
+
+        services_to_reload = list(services_to_reload)
+        services_to_reload.sort(key="nginx".__eq__)
+        if services_to_reload:
+            logger.info("Reloading services...")
+        for service in services_to_reload:
+            service = service.replace("__APP__", self.app)
+            service_reload_or_restart(service)
+
+    def _iterate(self, trigger=["option"]):
+        for panel in self.config.get("panels", []):
+            if "panel" in trigger:
+                yield (panel, None, panel)
+            for section in panel.get("sections", []):
+                if "section" in trigger:
+                    yield (panel, section, section)
+                if "option" in trigger:
+                    for option in section.get("options", []):
+                        yield (panel, section, option)
+
+
+class Question(object):
+    hide_user_input_in_prompt = False
+    operation_logger = None
+    pattern = None
+
+    def __init__(self, question, user_answers):
+        self.name = question["name"]
+        self.type = question.get("type", "string")
+        self.default = question.get("default", None)
+        self.current_value = question.get("current_value")
+        self.optional = question.get("optional", False)
+        self.choices = question.get("choices", [])
+        self.pattern = question.get("pattern", self.pattern)
+        self.ask = question.get("ask", {"en": self.name})
+        self.help = question.get("help")
+        self.value = user_answers.get(self.name)
+        self.redact = question.get("redact", False)
+
+        # Empty value is parsed as empty string
+        if self.default == "":
+            self.default = None
+
+    @staticmethod
+    def humanize(value, option={}):
+        return str(value)
+
+    @staticmethod
+    def normalize(value, option={}):
+        return value
+
+    def ask_if_needed(self):
+        for i in range(5):
+            # Display question if no value filled or if it's a readonly message
+            if Moulinette.interface.type == "cli" and os.isatty(1):
+                text_for_user_input_in_cli = self._format_text_for_user_input_in_cli()
+                if getattr(self, "readonly", False):
+                    Moulinette.display(text_for_user_input_in_cli)
+
+                elif self.value is None:
+                    prefill = ""
+                    if self.current_value is not None:
+                        prefill = self.humanize(self.current_value, self)
+                    elif self.default is not None:
+                        prefill = self.humanize(self.default, self)
+                    self.value = Moulinette.prompt(
+                        message=text_for_user_input_in_cli,
+                        is_password=self.hide_user_input_in_prompt,
+                        confirm=self.hide_user_input_in_prompt,
+                        prefill=prefill,
+                        is_multiline=(self.type == "text"),
+                    )
+
+            # Apply default value
+            class_default= getattr(self, "default_value", None)
+            if self.value in [None, ""] and \
+               (self.default is not None or class_default is not None):
+                self.value = (
+                    class_default
+                    if self.default is None
+                    else self.default
+                )
+
+            # Normalization
+            # This is done to enforce a certain formating like for boolean
+            self.value = self.normalize(self.value, self)
+
+            # Prevalidation
+            try:
+                self._prevalidate()
+            except YunohostValidationError as e:
+                # If in interactive cli, re-ask the current question
+                if i < 4 and Moulinette.interface.type == "cli" and os.isatty(1):
+                    logger.error(str(e))
+                    self.value = None
+                    continue
+
+                # Otherwise raise the ValidationError
+                raise
+
+            break
+        self.value = self._post_parse_value()
+
+        return (self.value, self.argument_type)
+
+    def _prevalidate(self):
+        if self.value in [None, ""] and not self.optional:
+            raise YunohostValidationError("app_argument_required", name=self.name)
+
+        # we have an answer, do some post checks
+        if self.value is not None:
+            if self.choices and self.value not in self.choices:
+                self._raise_invalid_answer()
+            if self.pattern and not re.match(self.pattern["regexp"], str(self.value)):
+                raise YunohostValidationError(
+                    self.pattern["error"],
+                    name=self.name,
+                    value=self.value,
+                )
+
+    def _raise_invalid_answer(self):
+        raise YunohostValidationError(
+            "app_argument_choice_invalid",
+            name=self.name,
+            value=self.value,
+            choices=", ".join(self.choices),
+        )
+
+    def _format_text_for_user_input_in_cli(self):
+        text_for_user_input_in_cli = _value_for_locale(self.ask)
+
+        if self.choices:
+            text_for_user_input_in_cli += " [{0}]".format(" | ".join(self.choices))
+
+        if self.help:
+            text_for_user_input_in_cli += ":\033[m"
+        if self.help:
+            text_for_user_input_in_cli += "\n - "
+            text_for_user_input_in_cli += _value_for_locale(self.help)
+        return text_for_user_input_in_cli
+
+    def _post_parse_value(self):
+        if not self.redact:
+            return self.value
+
+        # Tell the operation_logger to redact all password-type / secret args
+        # Also redact the % escaped version of the password that might appear in
+        # the 'args' section of metadata (relevant for password with non-alphanumeric char)
+        data_to_redact = []
+        if self.value and isinstance(self.value, str):
+            data_to_redact.append(self.value)
+        if self.current_value and isinstance(self.current_value, str):
+            data_to_redact.append(self.current_value)
+        data_to_redact += [
+            urllib.parse.quote(data)
+            for data in data_to_redact
+            if urllib.parse.quote(data) != data
+        ]
+        if self.operation_logger:
+            self.operation_logger.data_to_redact.extend(data_to_redact)
+        elif data_to_redact:
+            raise YunohostError(
+                f"Can't redact {self.name} because no operation logger available in the context",
+                raw_msg=True,
+            )
+
+        return self.value
+
+
+class StringQuestion(Question):
+    argument_type = "string"
+    default_value = ""
+
+
+class EmailQuestion(StringQuestion):
+    pattern = {
+        "regexp": r"^.+@.+",
+        "error": "config_validate_email"  # i18n: config_validate_email
+    }
+
+
+class URLQuestion(StringQuestion):
+    pattern = {
+        "regexp": r"^https?://.*$",
+        "error": "config_validate_url"  # i18n: config_validate_url
+    }
+
+
+class DateQuestion(StringQuestion):
+    pattern = {
+        "regexp": r"^\d{4}-\d\d-\d\d$",
+        "error": "config_validate_date"  # i18n: config_validate_date
+    }
+
+    def _prevalidate(self):
+        from datetime import datetime
+        super()._prevalidate()
+
+        if self.value not in [None, ""]:
+            try:
+                datetime.strptime(self.value, '%Y-%m-%d')
+            except ValueError:
+                raise YunohostValidationError("config_validate_date")
+
+
+class TimeQuestion(StringQuestion):
+    pattern = {
+        "regexp": r"^(1[12]|0?\d):[0-5]\d$",
+        "error": "config_validate_time"  # i18n: config_validate_time
+    }
+
+
+class ColorQuestion(StringQuestion):
+    pattern = {
+        "regexp": r"^#[ABCDEFabcdef\d]{3,6}$",
+        "error": "config_validate_color"  # i18n: config_validate_color
+    }
+
+
+class TagsQuestion(Question):
+    argument_type = "tags"
+
+    @staticmethod
+    def humanize(value, option={}):
+        if isinstance(value, list):
+            return ",".join(value)
+        return value
+
+    def _prevalidate(self):
+        values = self.value
+        if isinstance(values, str):
+            values = values.split(",")
+        elif values is None:
+            values = []
+        for value in values:
+            self.value = value
+            super()._prevalidate()
+        self.value = values
+
+
+class PasswordQuestion(Question):
+    hide_user_input_in_prompt = True
+    argument_type = "password"
+    default_value = ""
+    forbidden_chars = "{}"
+
+    def __init__(self, question, user_answers):
+        super().__init__(question, user_answers)
+        self.redact = True
+        if self.default is not None:
+            raise YunohostValidationError(
+                "app_argument_password_no_default", name=self.name
+            )
+
+    @staticmethod
+    def humanize(value, option={}):
+        if value:
+            return "********"  # Avoid to display the password on screen
+        return ""
+
+    def _prevalidate(self):
+        super()._prevalidate()
+
+        if self.value not in [None, ""]:
+            if any(char in self.value for char in self.forbidden_chars):
+                raise YunohostValidationError(
+                    "pattern_password_app", forbidden_chars=self.forbidden_chars
+                )
+
+            # If it's an optional argument the value should be empty or strong enough
+            from yunohost.utils.password import assert_password_is_strong_enough
+
+            assert_password_is_strong_enough("user", self.value)
+
+
+class PathQuestion(Question):
+    argument_type = "path"
+    default_value = ""
+
+
+class BooleanQuestion(Question):
+    argument_type = "boolean"
+    default_value = 0
+    yes_answers = ["1", "yes", "y", "true", "t", "on"]
+    no_answers = ["0", "no", "n", "false", "f", "off"]
+
+    @staticmethod
+    def humanize(value, option={}):
+        yes = option.get("yes", 1)
+        no = option.get("no", 0)
+        value = str(value).lower()
+        if value == str(yes).lower():
+            return "yes"
+        if value == str(no).lower():
+            return "no"
+        if value in BooleanQuestion.yes_answers:
+            return "yes"
+        if value in BooleanQuestion.no_answers:
+            return "no"
+
+        if value in ["none", ""]:
+            return ""
+
+        raise YunohostValidationError(
+            "app_argument_choice_invalid",
+            name=option.get("name", ""),
+            value=value,
+            choices="yes, no, y, n, 1, 0",
+        )
+
+    @staticmethod
+    def normalize(value, option={}):
+        yes = option.get("yes", 1)
+        no = option.get("no", 0)
+
+        if str(value).lower() in BooleanQuestion.yes_answers:
+            return yes
+
+        if str(value).lower() in BooleanQuestion.no_answers:
+            return no
+
+        if value in [None, ""]:
+            return None
+        raise YunohostValidationError(
+            "app_argument_choice_invalid",
+            name=option.get("name", ""),
+            value=value,
+            choices="yes, no, y, n, 1, 0",
+        )
+
+    def __init__(self, question, user_answers):
+        super().__init__(question, user_answers)
+        self.yes = question.get("yes", 1)
+        self.no = question.get("no", 0)
+        if self.default is None:
+            self.default = self.no
+
+    def _format_text_for_user_input_in_cli(self):
+        text_for_user_input_in_cli = super()._format_text_for_user_input_in_cli()
+
+        text_for_user_input_in_cli += " [yes | no]"
+
+        return text_for_user_input_in_cli
+
+    def get(self, key, default=None):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            return default
+
+
+class DomainQuestion(Question):
+    argument_type = "domain"
+
+    def __init__(self, question, user_answers):
+        from yunohost.domain import domain_list, _get_maindomain
+
+        super().__init__(question, user_answers)
+
+        if self.default is None:
+            self.default = _get_maindomain()
+
+        self.choices = domain_list()["domains"]
+
+    def _raise_invalid_answer(self):
+        raise YunohostValidationError(
+            "app_argument_invalid", name=self.name, error=m18n.n("domain_name_unknown", domain=self.value)
+        )
+
+
+class UserQuestion(Question):
+    argument_type = "user"
+
+    def __init__(self, question, user_answers):
+        from yunohost.user import user_list, user_info
+        from yunohost.domain import _get_maindomain
+
+        super().__init__(question, user_answers)
+        self.choices = user_list()["users"]
+        if self.default is None:
+            root_mail = "root@%s" % _get_maindomain()
+            for user in self.choices.keys():
+                if root_mail in user_info(user).get("mail-aliases", []):
+                    self.default = user
+                    break
+
+    def _raise_invalid_answer(self):
+        raise YunohostValidationError(
+            "app_argument_invalid",
+            name=self.name,
+            error=m18n.n("user_unknown", user=self.value),
+        )
+
+
+class NumberQuestion(Question):
+    argument_type = "number"
+    default_value = None
+
+    def __init__(self, question, user_answers):
+        super().__init__(question, user_answers)
+        self.min = question.get("min", None)
+        self.max = question.get("max", None)
+        self.step = question.get("step", None)
+
+    @staticmethod
+    def normalize(value, option={}):
+        if isinstance(value, int):
+            return value
+
+        if isinstance(value, str) and value.isdigit():
+            return int(value)
+
+        if value in [None, ""]:
+            return value
+
+        raise YunohostValidationError(
+            "app_argument_invalid", name=option.name, error=m18n.n("invalid_number")
+        )
+
+    def _prevalidate(self):
+        super()._prevalidate()
+        if self.value in [None, ""]:
+            return
+
+        if self.min is not None and int(self.value) < self.min:
+            raise YunohostValidationError(
+                "app_argument_invalid",
+                name=self.name,
+                error=m18n.n("invalid_number_min", min=self.min),
+            )
+
+        if self.max is not None and int(self.value) > self.max:
+            raise YunohostValidationError(
+                "app_argument_invalid",
+                name=self.name,
+                error=m18n.n("invalid_number_max", max=self.max),
+            )
+
+
+class DisplayTextQuestion(Question):
+    argument_type = "display_text"
+    readonly = True
+
+    def __init__(self, question, user_answers):
+        super().__init__(question, user_answers)
+
+        self.optional = True
+        self.style = question.get("style", "info" if question['type'] == 'alert' else '')
+
+    def _format_text_for_user_input_in_cli(self):
+        text = _value_for_locale(self.ask)
+
+        if self.style in ["success", "info", "warning", "danger"]:
+            color = {
+                "success": "green",
+                "info": "cyan",
+                "warning": "yellow",
+                "danger": "red",
+            }
+            text = m18n.g(self.style) if self.style != "danger" else m18n.n("danger")
+            return colorize(text, color[self.style]) + f" {text}"
+        else:
+            return text
+
+
+class FileQuestion(Question):
+    argument_type = "file"
+    upload_dirs = []
+
+    @classmethod
+    def clean_upload_dirs(cls):
+        # Delete files uploaded from API
+        if Moulinette.interface.type == "api":
+            for upload_dir in cls.upload_dirs:
+                if os.path.exists(upload_dir):
+                    shutil.rmtree(upload_dir)
+
+    def __init__(self, question, user_answers):
+        super().__init__(question, user_answers)
+        if question.get("accept"):
+            self.accept = question.get("accept")
+        else:
+            self.accept = ""
+        if Moulinette.interface.type == "api":
+            if user_answers.get(f"{self.name}[name]"):
+                self.value = {
+                    "content": self.value,
+                    "filename": user_answers.get(f"{self.name}[name]", self.name),
+                }
+        # If path file are the same
+        if self.value and str(self.value) == self.current_value:
+            self.value = None
+
+    def _prevalidate(self):
+        super()._prevalidate()
+        if (
+            isinstance(self.value, str)
+            and self.value
+            and not os.path.exists(self.value)
+        ):
+            raise YunohostValidationError(
+                "app_argument_invalid",
+                name=self.name,
+                error=m18n.n("file_does_not_exist", path=self.value),
+            )
+        if self.value in [None, ""] or not self.accept:
+            return
+
+        filename = self.value if isinstance(self.value, str) else self.value["filename"]
+        if "." not in filename or "." + filename.split(".")[-1] not in self.accept.replace(" ", "").split(","):
+            raise YunohostValidationError(
+                "app_argument_invalid",
+                name=self.name,
+                error=m18n.n(
+                    "file_extension_not_accepted", file=filename, accept=self.accept
+                ),
+            )
+
+    def _post_parse_value(self):
+        from base64 import b64decode
+
+        # Upload files from API
+        # A file arg contains a string with "FILENAME:BASE64_CONTENT"
+        if not self.value:
+            return self.value
+
+        if Moulinette.interface.type == "api":
+
+            upload_dir = tempfile.mkdtemp(prefix="tmp_configpanel_")
+            FileQuestion.upload_dirs += [upload_dir]
+            filename = self.value["filename"]
+            logger.debug(
+                f"Save uploaded file {self.value['filename']} from API into {upload_dir}"
+            )
+
+            # Filename is given by user of the API. For security reason, we have replaced
+            # os.path.join to avoid the user to be able to rewrite a file in filesystem
+            # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
+            file_path = os.path.normpath(upload_dir + "/" + filename)
+            if not file_path.startswith(upload_dir + "/"):
+                raise YunohostError(
+                    f"Filename '{filename}' received from the API got a relative parent path, which is forbidden",
+                    raw_msg=True,
+                )
+            i = 2
+            while os.path.exists(file_path):
+                file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
+                i += 1
+            content = self.value["content"]
+
+            write_to_file(file_path, b64decode(content), file_mode="wb")
+
+            self.value = file_path
+        return self.value
+
+
+ARGUMENTS_TYPE_PARSERS = {
+    "string": StringQuestion,
+    "text": StringQuestion,
+    "select": StringQuestion,
+    "tags": TagsQuestion,
+    "email": EmailQuestion,
+    "url": URLQuestion,
+    "date": DateQuestion,
+    "time": TimeQuestion,
+    "color": ColorQuestion,
+    "password": PasswordQuestion,
+    "path": PathQuestion,
+    "boolean": BooleanQuestion,
+    "domain": DomainQuestion,
+    "user": UserQuestion,
+    "number": NumberQuestion,
+    "range": NumberQuestion,
+    "display_text": DisplayTextQuestion,
+    "alert": DisplayTextQuestion,
+    "markdown": DisplayTextQuestion,
+    "file": FileQuestion,
+}
+
+
+def parse_args_in_yunohost_format(user_answers, argument_questions):
+    """Parse arguments store in either manifest.json or actions.json or from a
+    config panel against the user answers when they are present.
+
+    Keyword arguments:
+        user_answers -- a dictionnary of arguments from the user (generally
+                        empty in CLI, filed from the admin interface)
+        argument_questions -- the arguments description store in yunohost
+                              format from actions.json/toml, manifest.json/toml
+                              or config_panel.json/toml
+    """
+    parsed_answers_dict = OrderedDict()
+
+    for question in argument_questions:
+        question_class = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]
+        question = question_class(question, user_answers)
+
+        answer = question.ask_if_needed()
+        if answer is not None:
+            parsed_answers_dict[question.name] = answer
+
+    return parsed_answers_dict

src/yunohost/utils/error.py

@@ -59,4 +59,4 @@ class YunohostValidationError(YunohostError):
 
     def content(self):
 
-        return {"error": self.strerror, "error_key": self.key}
+        return {"error": self.strerror, "error_key": self.key, **self.kwargs}

src/yunohost/utils/i18n.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2018 YUNOHOST.ORG
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+from moulinette import m18n
+
+
+def _value_for_locale(values):
+    """
+    Return proper value for current locale
+
+    Keyword arguments:
+        values -- A dict of values associated to their locale
+
+    Returns:
+        An utf-8 encoded string
+
+    """
+    if not isinstance(values, dict):
+        return values
+
+    for lang in [m18n.locale, m18n.default_locale]:
+        try:
+            return values[lang]
+        except KeyError:
+            continue
+
+    # Fallback to first value
+    return list(values.values())[0]

tests/test_helpers.d/ynhtest_config.sh

@@ -0,0 +1,662 @@
+
+#################
+#  _ __  _   _  #
+# | '_ \| | | | #
+# | |_) | |_| | #
+# | .__/ \__, | #
+# | |     __/ | #
+# |_|    |___/  #
+#               #
+#################
+
+_read_py() {
+    local file="$1"
+    local key="$2"
+    python3 -c "exec(open('$file').read()); print($key)"
+}
+
+ynhtest_config_read_py() {
+
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.py"
+
+    cat << EOF > $dummy_dir/dummy.py
+# Some comment
+FOO = None
+ENABLED = False
+# TITLE = "Old title"
+TITLE = "Lorem Ipsum"
+THEME = "colib'ris"
+EMAIL = "root@example.com" # This is a comment without quotes
+PORT   = 1234 # This is a comment without quotes
+URL = 'https://yunohost.org'
+DICT = {}
+DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org"
+DICT['ldap_conf'] = {}
+DICT['ldap_conf']['user'] = "camille"
+# YNH_ICI
+DICT['TITLE'] = "Hello world"
+EOF
+
+    test "$(_read_py                   "$file" "FOO")" == "None"
+    test "$(ynh_read_var_in_file       "$file" "FOO")" == "None"
+
+    test "$(_read_py                   "$file" "ENABLED")" == "False"
+    test "$(ynh_read_var_in_file       "$file" "ENABLED")" == "False"
+
+    test "$(_read_py                   "$file" "TITLE")" == "Lorem Ipsum"
+    test "$(ynh_read_var_in_file       "$file" "TITLE")" == "Lorem Ipsum"
+
+    test "$(_read_py                   "$file" "THEME")" == "colib'ris"
+    test "$(ynh_read_var_in_file       "$file" "THEME")" == "colib'ris"
+
+    test "$(_read_py                   "$file" "EMAIL")" == "root@example.com"
+    test "$(ynh_read_var_in_file       "$file" "EMAIL")" == "root@example.com"
+
+    test "$(_read_py                   "$file" "PORT")" == "1234"
+    test "$(ynh_read_var_in_file       "$file" "PORT")" == "1234"
+
+    test "$(_read_py                   "$file" "URL")" == "https://yunohost.org"
+    test "$(ynh_read_var_in_file       "$file" "URL")" == "https://yunohost.org"
+
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org"
+    
+    test "$(ynh_read_var_in_file   "$file" "user")" == "camille"
+    
+    test "$(ynh_read_var_in_file       "$file" "TITLE" "YNH_ICI")" == "Hello world"
+
+    ! _read_py                         "$file" "NONEXISTENT"
+    test "$(ynh_read_var_in_file       "$file" "NONEXISTENT")" == "YNH_NULL"
+
+    ! _read_py                         "$file" "ENABLE"
+    test "$(ynh_read_var_in_file       "$file" "ENABLE")" == "YNH_NULL"
+}
+
+ynhtest_config_write_py() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.py"
+
+    cat << EOF > $dummy_dir/dummy.py
+# Some comment
+FOO = None
+ENABLED = False
+# TITLE = "Old title"
+TITLE = "Lorem Ipsum"
+THEME = "colib'ris"
+EMAIL = "root@example.com" # This is a comment without quotes
+PORT   = 1234 # This is a comment without quotes
+URL = 'https://yunohost.org'
+DICT = {}
+DICT['ldap_base'] = "ou=users,dc=yunohost,dc=org"
+# YNH_ICI
+DICT['TITLE'] = "Hello world"
+EOF
+
+    ynh_write_var_in_file              "$file" "FOO"      "bar"
+    test "$(_read_py                   "$file" "FOO")" == "bar"
+    test "$(ynh_read_var_in_file       "$file" "FOO")" == "bar"
+
+    ynh_write_var_in_file              "$file" "ENABLED"      "True"
+    test "$(_read_py                   "$file" "ENABLED")" == "True"
+    test "$(ynh_read_var_in_file       "$file" "ENABLED")" == "True"
+
+    ynh_write_var_in_file              "$file" "TITLE"      "Foo Bar"
+    test "$(_read_py                   "$file" "TITLE")" == "Foo Bar"
+    test "$(ynh_read_var_in_file       "$file" "TITLE")" == "Foo Bar"
+
+    ynh_write_var_in_file              "$file" "THEME"      "super-awesome-theme"
+    test "$(_read_py                   "$file" "THEME")" == "super-awesome-theme"
+    test "$(ynh_read_var_in_file       "$file" "THEME")" == "super-awesome-theme"
+
+    ynh_write_var_in_file              "$file" "EMAIL"      "sam@domain.tld"
+    test "$(_read_py                   "$file" "EMAIL")" == "sam@domain.tld"
+    test "$(ynh_read_var_in_file       "$file" "EMAIL")" == "sam@domain.tld"
+
+    ynh_write_var_in_file              "$file" "PORT"      "5678"
+    test "$(_read_py                   "$file" "PORT")" == "5678"
+    test "$(ynh_read_var_in_file       "$file" "PORT")" == "5678"
+
+    ynh_write_var_in_file              "$file" "URL"      "https://domain.tld/foobar"
+    test "$(_read_py                   "$file" "URL")" == "https://domain.tld/foobar"
+    test "$(ynh_read_var_in_file       "$file" "URL")" == "https://domain.tld/foobar"
+
+    ynh_write_var_in_file              "$file" "ldap_base"      "ou=users,dc=yunohost,dc=org"
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org"
+    
+    ynh_write_var_in_file              "$file" "TITLE"          "YOLO" "YNH_ICI"
+    test "$(ynh_read_var_in_file       "$file" "TITLE" "YNH_ICI")" == "YOLO"
+
+    ! ynh_write_var_in_file              "$file" "NONEXISTENT" "foobar"
+    ! _read_py                         "$file" "NONEXISTENT"
+    test "$(ynh_read_var_in_file       "$file" "NONEXISTENT")" == "YNH_NULL"
+
+    ! ynh_write_var_in_file              "$file" "ENABLE" "foobar"
+    ! _read_py                         "$file" "ENABLE"
+    test "$(ynh_read_var_in_file       "$file" "ENABLE")" == "YNH_NULL"
+
+}
+
+###############
+#  _       _  #
+# (_)     (_) #
+#  _ _ __  _  #
+# | | '_ \| | #
+# | | | | | | #
+# |_|_| |_|_| #
+#             #
+###############
+
+_read_ini() {
+    local file="$1"
+    local key="$2"
+    python3 -c "import configparser; c = configparser.ConfigParser(); c.read('$file'); print(c['main']['$key'])"
+}
+
+ynhtest_config_read_ini() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.ini"
+
+    cat << EOF > $file
+# Some comment
+; Another comment
+[main]
+foo = null
+enabled =    False
+# title = Old title
+title =    Lorem Ipsum
+theme = colib'ris
+email = root@example.com ; This is a comment without quotes
+port =     1234 ; This is a comment without quotes
+url = https://yunohost.org
+[dict]
+    ldap_base = ou=users,dc=yunohost,dc=org
+EOF
+
+    test "$(_read_ini                  "$file" "foo")" == "null"
+    test "$(ynh_read_var_in_file       "$file" "foo")" == "null"
+
+    test "$(_read_ini                  "$file" "enabled")" == "False"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "False"
+
+    test "$(_read_ini                  "$file" "title")" == "Lorem Ipsum"
+    test "$(ynh_read_var_in_file       "$file" "title")" == "Lorem Ipsum"
+
+    test "$(_read_ini                  "$file" "theme")" == "colib'ris"
+    test "$(ynh_read_var_in_file       "$file" "theme")" == "colib'ris"
+
+    #test "$(_read_ini                  "$file" "email")" == "root@example.com"
+    test "$(ynh_read_var_in_file       "$file" "email")" == "root@example.com"
+
+    #test "$(_read_ini                  "$file" "port")" == "1234"
+    test "$(ynh_read_var_in_file       "$file" "port")" == "1234"
+
+    test "$(_read_ini                  "$file" "url")" == "https://yunohost.org"
+    test "$(ynh_read_var_in_file       "$file" "url")" == "https://yunohost.org"
+
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org"
+
+    ! _read_ini                        "$file" "nonexistent"
+    test "$(ynh_read_var_in_file       "$file" "nonexistent")" == "YNH_NULL"
+
+    ! _read_ini                        "$file" "enable"
+    test "$(ynh_read_var_in_file       "$file" "enable")" == "YNH_NULL"
+
+}
+
+ynhtest_config_write_ini() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.ini"
+
+    cat << EOF > $file
+# Some comment
+; Another comment
+[main]
+foo = null
+enabled =    False
+# title = Old title
+title =    Lorem Ipsum
+theme = colib'ris
+email = root@example.com # This is a comment without quotes
+port =     1234 # This is a comment without quotes
+url = https://yunohost.org
+[dict]
+    ldap_base = ou=users,dc=yunohost,dc=org
+EOF
+
+    ynh_write_var_in_file              "$file" "foo"      "bar"
+    test "$(_read_ini                  "$file" "foo")" == "bar"
+    test "$(ynh_read_var_in_file       "$file" "foo")" == "bar"
+
+    ynh_write_var_in_file              "$file" "enabled"      "True"
+    test "$(_read_ini                  "$file" "enabled")" == "True"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "True"
+
+    ynh_write_var_in_file              "$file" "title"      "Foo Bar"
+    test "$(_read_ini                  "$file" "title")" == "Foo Bar"
+    test "$(ynh_read_var_in_file       "$file" "title")" == "Foo Bar"
+
+    ynh_write_var_in_file              "$file" "theme"      "super-awesome-theme"
+    test "$(_read_ini                  "$file" "theme")" == "super-awesome-theme"
+    test "$(ynh_read_var_in_file       "$file" "theme")" == "super-awesome-theme"
+
+    ynh_write_var_in_file              "$file" "email"      "sam@domain.tld"
+    test "$(_read_ini                  "$file" "email")" == "sam@domain.tld # This is a comment without quotes"
+    test "$(ynh_read_var_in_file       "$file" "email")" == "sam@domain.tld"
+
+    ynh_write_var_in_file              "$file" "port"      "5678"
+    test "$(_read_ini                  "$file" "port")" == "5678 # This is a comment without quotes"
+    test "$(ynh_read_var_in_file       "$file" "port")" == "5678"
+
+    ynh_write_var_in_file              "$file" "url"      "https://domain.tld/foobar"
+    test "$(_read_ini                  "$file" "url")" == "https://domain.tld/foobar"
+    test "$(ynh_read_var_in_file       "$file" "url")" == "https://domain.tld/foobar"
+
+    ynh_write_var_in_file              "$file" "ldap_base"      "ou=users,dc=yunohost,dc=org"
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org"
+
+    ! ynh_write_var_in_file              "$file" "nonexistent" "foobar"
+    ! _read_ini                        "$file" "nonexistent"
+    test "$(ynh_read_var_in_file       "$file" "nonexistent")" == "YNH_NULL"
+
+    ! ynh_write_var_in_file              "$file" "enable" "foobar"
+    ! _read_ini                        "$file" "enable"
+    test "$(ynh_read_var_in_file       "$file" "enable")" == "YNH_NULL"
+
+}
+
+#############################
+#                        _  #
+#                       | | #
+#  _   _  __ _ _ __ ___ | | #
+# | | | |/ _` | '_ ` _ \| | #
+# | |_| | (_| | | | | | | | #
+#  \__, |\__,_|_| |_| |_|_| #
+#   __/ |                   #
+#  |___/                    #
+#                           #
+#############################
+
+_read_yaml() {
+    local file="$1"
+    local key="$2"
+    python3 -c "import yaml; print(yaml.safe_load(open('$file'))['$key'])"
+}
+
+ynhtest_config_read_yaml() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.yml"
+
+    cat << EOF > $file
+# Some comment
+foo:
+enabled: false
+# title: old title
+title: Lorem Ipsum
+theme: colib'ris
+email: root@example.com # This is a comment without quotes
+port: 1234 # This is a comment without quotes
+url: https://yunohost.org
+dict:
+   ldap_base: ou=users,dc=yunohost,dc=org
+EOF
+
+    test "$(_read_yaml                 "$file" "foo")" == "None"
+    test "$(ynh_read_var_in_file       "$file" "foo")" == ""
+
+    test "$(_read_yaml                 "$file" "enabled")" == "False"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "false"
+
+    test "$(_read_yaml                 "$file" "title")" == "Lorem Ipsum"
+    test "$(ynh_read_var_in_file       "$file" "title")" == "Lorem Ipsum"
+
+    test "$(_read_yaml                 "$file" "theme")" == "colib'ris"
+    test "$(ynh_read_var_in_file       "$file" "theme")" == "colib'ris"
+
+    test "$(_read_yaml                 "$file" "email")" == "root@example.com"
+    test "$(ynh_read_var_in_file       "$file" "email")" == "root@example.com"
+
+    test "$(_read_yaml                 "$file" "port")" == "1234"
+    test "$(ynh_read_var_in_file       "$file" "port")" == "1234"
+
+    test "$(_read_yaml                 "$file" "url")" == "https://yunohost.org"
+    test "$(ynh_read_var_in_file       "$file" "url")" == "https://yunohost.org"
+
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org"
+
+    ! _read_yaml                       "$file" "nonexistent"
+    test "$(ynh_read_var_in_file       "$file" "nonexistent")" == "YNH_NULL"
+
+    ! _read_yaml                       "$file" "enable"
+    test "$(ynh_read_var_in_file       "$file" "enable")" == "YNH_NULL"
+}
+
+
+ynhtest_config_write_yaml() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.yml"
+
+    cat << EOF > $file
+# Some comment
+foo:
+enabled: false
+# title: old title
+title: Lorem Ipsum
+theme: colib'ris
+email: root@example.com # This is a comment without quotes
+port: 1234 # This is a comment without quotes
+url: https://yunohost.org
+dict:
+   ldap_base: ou=users,dc=yunohost,dc=org
+EOF
+
+    ynh_write_var_in_file              "$file" "foo"      "bar"
+    # cat $dummy_dir/dummy.yml   # to debug
+    ! test "$(_read_yaml                 "$file" "foo")" == "bar" # writing broke the yaml syntax... "foo:bar" (no space aftr :)
+    test "$(ynh_read_var_in_file       "$file" "foo")" == "bar"
+
+    ynh_write_var_in_file              "$file" "enabled"      "true"
+    test "$(_read_yaml                 "$file" "enabled")" == "True"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "true"
+
+    ynh_write_var_in_file              "$file" "title"      "Foo Bar"
+    test "$(_read_yaml                 "$file" "title")" == "Foo Bar"
+    test "$(ynh_read_var_in_file       "$file" "title")" == "Foo Bar"
+
+    ynh_write_var_in_file              "$file" "theme"      "super-awesome-theme"
+    test "$(_read_yaml                 "$file" "theme")" == "super-awesome-theme"
+    test "$(ynh_read_var_in_file       "$file" "theme")" == "super-awesome-theme"
+
+    ynh_write_var_in_file              "$file" "email"      "sam@domain.tld"
+    test "$(_read_yaml                 "$file" "email")" == "sam@domain.tld"
+    test "$(ynh_read_var_in_file       "$file" "email")" == "sam@domain.tld"
+
+    ynh_write_var_in_file              "$file" "port"      "5678"
+    test "$(_read_yaml                 "$file" "port")" == "5678"
+    test "$(ynh_read_var_in_file       "$file" "port")" == "5678"
+
+    ynh_write_var_in_file              "$file" "url"      "https://domain.tld/foobar"
+    test "$(_read_yaml                 "$file" "url")" == "https://domain.tld/foobar"
+    test "$(ynh_read_var_in_file       "$file" "url")" == "https://domain.tld/foobar"
+
+    ynh_write_var_in_file              "$file" "ldap_base"      "ou=foobar,dc=domain,dc=tld"
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld"
+
+    ! ynh_write_var_in_file              "$file" "nonexistent"      "foobar"
+    test "$(ynh_read_var_in_file       "$file" "nonexistent")" == "YNH_NULL"
+
+    ! ynh_write_var_in_file              "$file" "enable"       "foobar"
+    test "$(ynh_read_var_in_file       "$file" "enable")"  == "YNH_NULL"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "true"
+}
+
+#########################
+#    _                  #
+#   (_)                 #
+#    _ ___  ___  _ __   #
+#   | / __|/ _ \| '_ \  #
+#   | \__ \ (_) | | | | #
+#   | |___/\___/|_| |_| #
+#  _/ |                 #
+# |__/                  #
+#                       #
+#########################
+
+_read_json() {
+    local file="$1"
+    local key="$2"
+    python3 -c "import json; print(json.load(open('$file'))['$key'])"
+}
+
+ynhtest_config_read_json() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.json"
+
+    cat << EOF > $file
+{
+     "foo": null,
+     "enabled":     false,
+     "title": "Lorem Ipsum",
+     "theme": "colib'ris",
+     "email": "root@example.com",
+       "port": 1234,
+     "url": "https://yunohost.org",
+     "dict": {
+         "ldap_base": "ou=users,dc=yunohost,dc=org"
+     }
+}
+EOF
+
+
+    test "$(_read_json                 "$file" "foo")" == "None"
+    test "$(ynh_read_var_in_file       "$file" "foo")" == "null"
+
+    test "$(_read_json                 "$file" "enabled")" == "False"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "false"
+
+    test "$(_read_json                 "$file" "title")" == "Lorem Ipsum"
+    test "$(ynh_read_var_in_file       "$file" "title")" == "Lorem Ipsum"
+
+    test "$(_read_json                 "$file" "theme")" == "colib'ris"
+    test "$(ynh_read_var_in_file       "$file" "theme")" == "colib'ris"
+
+    test "$(_read_json                 "$file" "email")" == "root@example.com"
+    test "$(ynh_read_var_in_file       "$file" "email")" == "root@example.com"
+
+    test "$(_read_json                 "$file" "port")" == "1234"
+    test "$(ynh_read_var_in_file       "$file" "port")" == "1234"
+
+    test "$(_read_json                 "$file" "url")" == "https://yunohost.org"
+    test "$(ynh_read_var_in_file       "$file" "url")" == "https://yunohost.org"
+
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org"
+
+    ! _read_json                       "$file" "nonexistent"
+    test "$(ynh_read_var_in_file       "$file" "nonexistent")" == "YNH_NULL"
+
+    ! _read_json                       "$file" "enable"
+    test "$(ynh_read_var_in_file       "$file" "enable")" == "YNH_NULL"
+}
+
+
+ynhtest_config_write_json() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.json"
+
+    cat << EOF > $file
+{
+     "foo": null,
+     "enabled":     false,
+     "title": "Lorem Ipsum",
+     "theme": "colib'ris",
+     "email": "root@example.com",
+       "port": 1234,
+     "url": "https://yunohost.org",
+     "dict": {
+         "ldap_base": "ou=users,dc=yunohost,dc=org"
+     }
+}
+EOF
+
+    ynh_write_var_in_file              "$file" "foo"      "bar"
+    cat $file
+    test "$(_read_json                 "$file" "foo")" == "bar"
+    test "$(ynh_read_var_in_file       "$file" "foo")" == "bar"
+
+    ynh_write_var_in_file              "$file" "enabled"      "true"
+    cat $file
+    test "$(_read_json                 "$file" "enabled")" == "true"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "true"
+
+    ynh_write_var_in_file              "$file" "title"      "Foo Bar"
+    cat $file
+    test "$(_read_json                 "$file" "title")" == "Foo Bar"
+    test "$(ynh_read_var_in_file       "$file" "title")" == "Foo Bar"
+
+    ynh_write_var_in_file              "$file" "theme"      "super-awesome-theme"
+    cat $file
+    test "$(_read_json                 "$file" "theme")" == "super-awesome-theme"
+    test "$(ynh_read_var_in_file       "$file" "theme")" == "super-awesome-theme"
+
+    ynh_write_var_in_file              "$file" "email"      "sam@domain.tld"
+    cat $file
+    test "$(_read_json                 "$file" "email")" == "sam@domain.tld"
+    test "$(ynh_read_var_in_file       "$file" "email")" == "sam@domain.tld"
+
+    ynh_write_var_in_file              "$file" "port"      "5678"
+    test "$(_read_json                 "$file" "port")" == "5678"
+    test "$(ynh_read_var_in_file       "$file" "port")" == "5678"
+
+    ynh_write_var_in_file              "$file" "url"      "https://domain.tld/foobar"
+    test "$(_read_json                 "$file" "url")" == "https://domain.tld/foobar"
+    test "$(ynh_read_var_in_file       "$file" "url")" == "https://domain.tld/foobar"
+
+    ynh_write_var_in_file              "$file" "ldap_base"      "ou=foobar,dc=domain,dc=tld"
+    test "$(ynh_read_var_in_file       "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld"
+
+    ! ynh_write_var_in_file              "$file" "nonexistent"      "foobar"
+    test "$(ynh_read_var_in_file       "$file" "nonexistent")" == "YNH_NULL"
+
+    ! ynh_write_var_in_file              "$file" "enable"       "foobar"
+    test "$(ynh_read_var_in_file       "$file" "enable")"  == "YNH_NULL"
+    test "$(ynh_read_var_in_file       "$file" "enabled")" == "true"
+}
+
+#######################
+#        _            #
+#       | |           #
+#  _ __ | |__  _ __   #
+# | '_ \| '_ \| '_ \  #
+# | |_) | | | | |_) | #
+# | .__/|_| |_| .__/  #
+# | |         | |     #
+# |_|         |_|     #
+#                     #
+#######################
+
+_read_php() {
+    local file="$1"
+    local key="$2"
+    php -r "include '$file'; echo var_export(\$$key);" | sed "s/^'//g" | sed "s/'$//g"
+}
+
+ynhtest_config_read_php() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.php"
+
+    cat << EOF > $file
+<?php
+  // Some comment
+  \$foo = NULL;
+  \$enabled = false;
+  // \$title = "old title";
+  \$title = "Lorem Ipsum";
+   \$theme = "colib'ris";
+  \$email = "root@example.com"; // This is a comment without quotes
+  \$port = 1234; // This is a second comment without quotes
+  \$url = "https://yunohost.org";
+  \$dict = [
+     'ldap_base' => "ou=users,dc=yunohost,dc=org",
+     'ldap_conf' => []
+  ];
+  \$dict['ldap_conf']['user'] = 'camille';
+  const DB_HOST       = 'localhost';
+?>
+EOF
+
+    test "$(_read_php              "$file" "foo")" == "NULL"
+    test "$(ynh_read_var_in_file   "$file" "foo")" == "NULL" 
+
+    test "$(_read_php              "$file" "enabled")" == "false"
+    test "$(ynh_read_var_in_file   "$file" "enabled")" == "false" 
+
+    test "$(_read_php              "$file" "title")" == "Lorem Ipsum"
+    test "$(ynh_read_var_in_file   "$file" "title")" == "Lorem Ipsum"
+
+    test "$(_read_php              "$file" "theme")" == "colib\\'ris"
+    test "$(ynh_read_var_in_file   "$file" "theme")" == "colib'ris"
+
+    test "$(_read_php              "$file" "email")" == "root@example.com"
+    test "$(ynh_read_var_in_file   "$file" "email")" == "root@example.com"
+
+    test "$(_read_php              "$file" "port")" == "1234"
+    test "$(ynh_read_var_in_file   "$file" "port")" == "1234"
+
+    test "$(_read_php              "$file" "url")" == "https://yunohost.org"
+    test "$(ynh_read_var_in_file   "$file" "url")" == "https://yunohost.org"
+
+    test "$(ynh_read_var_in_file   "$file" "ldap_base")" == "ou=users,dc=yunohost,dc=org"
+    
+    test "$(ynh_read_var_in_file   "$file" "user")" == "camille"
+    
+    test "$(ynh_read_var_in_file   "$file" "DB_HOST")" == "localhost"
+
+    ! _read_php                    "$file" "nonexistent"
+    test "$(ynh_read_var_in_file   "$file" "nonexistent")" == "YNH_NULL"
+
+    ! _read_php                    "$file" "enable"
+    test "$(ynh_read_var_in_file   "$file" "enable")" == "YNH_NULL"
+}
+
+
+ynhtest_config_write_php() {
+    local dummy_dir="$(mktemp -d -p $VAR_WWW)"
+    file="$dummy_dir/dummy.php"
+
+    cat << EOF > $file
+<?php
+  // Some comment
+  \$foo = NULL;
+  \$enabled = false;
+  // \$title = "old title";
+  \$title = "Lorem Ipsum";
+   \$theme = "colib'ris";
+  \$email = "root@example.com"; // This is a comment without quotes
+  \$port = 1234; // This is a comment without quotes
+  \$url = "https://yunohost.org";
+  \$dict = [
+     'ldap_base' => "ou=users,dc=yunohost,dc=org",
+  ];
+?>
+EOF
+
+    ynh_write_var_in_file          "$file" "foo"      "bar"
+    test "$(_read_php              "$file" "foo")" == "bar"
+    test "$(ynh_read_var_in_file   "$file" "foo")" == "bar"
+
+    ynh_write_var_in_file          "$file" "enabled"      "true"
+    test "$(_read_php              "$file" "enabled")" == "true"
+    test "$(ynh_read_var_in_file   "$file" "enabled")" == "true"
+
+    ynh_write_var_in_file          "$file" "title"      "Foo Bar"
+    cat $file
+    test "$(_read_php              "$file" "title")" == "Foo Bar"
+    test "$(ynh_read_var_in_file   "$file" "title")" == "Foo Bar"
+
+    ynh_write_var_in_file          "$file" "theme"      "super-awesome-theme"
+    cat $file
+    test "$(_read_php              "$file" "theme")" == "super-awesome-theme"
+    test "$(ynh_read_var_in_file   "$file" "theme")" == "super-awesome-theme"
+
+    ynh_write_var_in_file          "$file" "email"      "sam@domain.tld"
+    cat $file
+    test "$(_read_php              "$file" "email")" == "sam@domain.tld"
+    test "$(ynh_read_var_in_file   "$file" "email")" == "sam@domain.tld"
+
+    ynh_write_var_in_file          "$file" "port"      "5678"
+    test "$(_read_php              "$file" "port")" == "5678"
+    test "$(ynh_read_var_in_file   "$file" "port")" == "5678"
+
+    ynh_write_var_in_file          "$file" "url"      "https://domain.tld/foobar"
+    test "$(_read_php              "$file" "url")" == "https://domain.tld/foobar"
+    test "$(ynh_read_var_in_file   "$file" "url")" == "https://domain.tld/foobar"
+
+    ynh_write_var_in_file          "$file" "ldap_base"      "ou=foobar,dc=domain,dc=tld"
+    test "$(ynh_read_var_in_file   "$file" "ldap_base")" == "ou=foobar,dc=domain,dc=tld"
+
+    ! ynh_write_var_in_file          "$file" "nonexistent"      "foobar"
+    test "$(ynh_read_var_in_file   "$file" "nonexistent")" == "YNH_NULL"
+
+    ! ynh_write_var_in_file          "$file" "enable"       "foobar"
+    test "$(ynh_read_var_in_file   "$file" "enable")"  == "YNH_NULL"
+    test "$(ynh_read_var_in_file   "$file" "enabled")" == "true"
+}