From a26a024092de78c7711f67f5bacf3297f1253c63 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 11 May 2020 19:13:10 +0200
Subject: [PATCH 01/51] [wip] Allow file upload from config-panel

---
 src/yunohost/app.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 2ca931a90..32bb1ece3 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -33,6 +33,7 @@ import re
 import subprocess
 import glob
 import urllib.parse
+import base64
 import tempfile
 from collections import OrderedDict
 
@@ -1867,6 +1868,7 @@ def app_config_apply(operation_logger, app, args):
     }
     args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
 
+    upload_dir = None
     for tab in config_panel.get("panel", []):
         tab_id = tab["id"]  # this makes things easier to debug on crash
         for section in tab.get("sections", []):
@@ -1878,6 +1880,23 @@ def app_config_apply(operation_logger, app, args):
                 ).upper()
 
                 if generated_name in args:
+                    # Upload files from API
+                    # A file arg contains a string with "FILENAME:BASE64_CONTENT"
+                    if option["type"] == "file" and msettings.get('interface') == 'api':
+                        if upload_dir is None:
+                            upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
+                        filename, args[generated_name] = args[generated_name].split(':')
+                        logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
+                        file_path = os.join(upload_dir, filename)
+                        try:
+                            with open(file_path, 'wb') as f:
+                                f.write(args[generated_name])
+                        except IOError as e:
+                            raise YunohostError("cannot_write_file", file=file_path, error=str(e))
+                        except Exception as e:
+                            raise YunohostError("error_writing_file", file=file_path, error=str(e))
+                        args[generated_name] = file_path
+
                     logger.debug(
                         "include into env %s=%s", generated_name, args[generated_name]
                     )
@@ -1899,6 +1918,11 @@ def app_config_apply(operation_logger, app, args):
         env=env,
     )[0]
 
+    # Delete files uploaded from API
+    if msettings.get('interface') == 'api':
+        if upload_dir is not None:
+            shutil.rmtree(upload_dir)
+
     if return_code != 0:
         msg = (
             "'script/config apply' return value code: %s (considered as an error)"

From 4939bbeb2e4b7a0362ee55d955fa33271bcf8c50 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 8 Jun 2020 19:18:22 +0200
Subject: [PATCH 02/51] [fix] Several files with same name

---
 src/yunohost/app.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 32bb1ece3..ae6accab0 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1882,15 +1882,21 @@ def app_config_apply(operation_logger, app, args):
                 if generated_name in args:
                     # Upload files from API
                     # A file arg contains a string with "FILENAME:BASE64_CONTENT"
-                    if option["type"] == "file" and msettings.get('interface') == 'api':
+                    if 'type' in option and option["type"] == "file" \
+                       and msettings.get('interface') == 'api':
                         if upload_dir is None:
                             upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
-                        filename, args[generated_name] = args[generated_name].split(':')
+                        filename = args[generated_name + '[name]']
+                        content = args[generated_name]
                         logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
-                        file_path = os.join(upload_dir, filename)
+                        file_path = os.path.join(upload_dir, filename)
+                        i = 2
+                        while os.path.exists(file_path):
+                            file_path = os.path.join(upload_dir, filename + (".%d" % i))
+                            i += 1
                         try:
                             with open(file_path, 'wb') as f:
-                                f.write(args[generated_name])
+                                f.write(content.decode("base64"))
                         except IOError as e:
                             raise YunohostError("cannot_write_file", file=file_path, error=str(e))
                         except Exception as e:
@@ -1907,7 +1913,7 @@ def app_config_apply(operation_logger, app, args):
     # for debug purpose
     for key in args:
         if key not in env:
-            logger.warning(
+            logger.debug(
                 "Ignore key '%s' from arguments because it is not in the config", key
             )
 

From 3bc45b5672f332e7cdbe314c756fcf4aac74e11f Mon Sep 17 00:00:00 2001
From: "ljf (zamentur)" <zamentur@users.noreply.github.com>
Date: Wed, 7 Oct 2020 00:31:20 +0200
Subject: [PATCH 03/51] [enh] Replace os.path.join to improve security

---
 src/yunohost/app.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index ae6accab0..f017521d2 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1889,10 +1889,14 @@ def app_config_apply(operation_logger, app, args):
                         filename = args[generated_name + '[name]']
                         content = args[generated_name]
                         logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
-                        file_path = os.path.join(upload_dir, filename)
+                        
+                        # Filename is given by user of the API. For security reason, we have replaced 
+                        # os.path.join to avoid the user to be able to rewrite a file in filesystem
+                        # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
+                        file_path = os.path.normpath(upload_dir + "/" + filename)
                         i = 2
                         while os.path.exists(file_path):
-                            file_path = os.path.join(upload_dir, filename + (".%d" % i))
+                            file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
                             i += 1
                         try:
                             with open(file_path, 'wb') as f:

From a5508b1db45d2f5ae94578f44b6026fd5b45d017 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 31 May 2021 16:32:19 +0200
Subject: [PATCH 04/51] [fix] Base64 python3 change

---
 src/yunohost/app.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index f017521d2..49033d8b4 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1845,7 +1845,7 @@ def app_config_apply(operation_logger, app, args):
     logger.warning(m18n.n("experimental_feature"))
 
     from yunohost.hook import hook_exec
-
+    from base64 import b64decode
     installed = _is_installed(app)
     if not installed:
         raise YunohostValidationError(
@@ -1889,8 +1889,8 @@ def app_config_apply(operation_logger, app, args):
                         filename = args[generated_name + '[name]']
                         content = args[generated_name]
                         logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
-                        
-                        # Filename is given by user of the API. For security reason, we have replaced 
+
+                        # Filename is given by user of the API. For security reason, we have replaced
                         # os.path.join to avoid the user to be able to rewrite a file in filesystem
                         # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
                         file_path = os.path.normpath(upload_dir + "/" + filename)
@@ -1900,7 +1900,7 @@ def app_config_apply(operation_logger, app, args):
                             i += 1
                         try:
                             with open(file_path, 'wb') as f:
-                                f.write(content.decode("base64"))
+                                f.write(b64decode(content))
                         except IOError as e:
                             raise YunohostError("cannot_write_file", file=file_path, error=str(e))
                         except Exception as e:

From 27ba82bd307ae28268f7e56c1b3a6a40060c62e8 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 1 Jun 2021 00:41:37 +0200
Subject: [PATCH 05/51] [enh] Add configpanel helpers

---
 data/helpers.d/configpanel | 259 +++++++++++++++++++++++++++++++++++++
 1 file changed, 259 insertions(+)
 create mode 100644 data/helpers.d/configpanel

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
new file mode 100644
index 000000000..f648826e4
--- /dev/null
+++ b/data/helpers.d/configpanel
@@ -0,0 +1,259 @@
+#!/bin/bash
+
+ynh_lowerdot_to_uppersnake() {
+    local lowerdot
+    lowerdot=$(echo "$1" | cut -d= -f1 | sed "s/\./_/g")
+    echo "${lowerdot^^}"
+}
+
+# Get a value from heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_value_get --file=PATH --key=KEY
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to get
+#
+# This helpers match several var affectation use case in several languages
+# We don't use jq or equivalent to keep comments and blank space in files
+# This helpers work line by line, it is not able to work correctly 
+# if you have several identical keys in your files
+# 
+# Example of line this helpers can managed correctly
+# .yml
+#     title: YunoHost documentation
+#     email: 'yunohost@yunohost.org'
+# .json
+#     "theme": "colib'ris",
+#     "port": 8102
+#     "some_boolean":     false,
+#     "user": null
+# .ini
+#     some_boolean = On
+#     action = "Clear"
+#     port = 20
+# .php
+#     $user=
+#     user => 20
+# .py
+#     USER = 8102
+#     user = 'https://donate.local'
+#     CUSTOM['user'] = 'YunoHost'
+# Requires YunoHost version 4.3 or higher.
+ynh_value_get() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fk
+    local -A args_array=( [f]=file= [k]=key= )
+    local file
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    
+    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    
+    local first_char="${crazy_value:0:1}"
+    if [[ "$first_char" == '"' ]] ; then
+        echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1  | sed 's/\\"/"/g'
+    elif [[ "$first_char" == "'" ]] ; then
+        echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1  | sed "s/\\\\'/'/g"
+    else
+        echo "$crazy_value"
+    fi
+}
+
+# Set a value into heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_value_set --file=PATH --key=KEY --value=VALUE
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to set
+# | arg: -v, --value=     - the value to set
+#
+# Requires YunoHost version 4.3 or higher.
+ynh_value_set() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fkv
+    local -A args_array=( [f]=file= [k]=key= [v]=value=)
+    local file
+    local key
+    local value
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    
+    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    local first_char="${crazy_value:0:1}"
+    if [[ "$first_char" == '"' ]] ; then
+        value="$(echo "$value" | sed 's/"/\\"/g')"
+        sed -ri "s%^(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+    elif [[ "$first_char" == "'" ]] ; then
+        value="$(echo "$value" | sed "s/'/\\\\'/g")"
+        sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+    else
+        if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
+            value="\"$(echo "$value" | sed 's/"/\\"/g')\""
+        fi
+        sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+    fi
+}
+
+_ynh_panel_get() {
+    
+    # From settings
+    local params_sources
+    params_sources=`python3 << EOL
+import toml
+from collections import OrderedDict
+with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f:
+    file_content = f.read()
+loaded_toml = toml.loads(file_content, _dict=OrderedDict)
+
+for panel_name,panel in loaded_toml.items():
+    if isinstance(panel, dict):
+        for section_name, section in panel.items():
+            if isinstance(section, dict):
+                for name, param in section.items():
+                    if isinstance(param, dict) and param.get('source', '') == 'settings':
+                        print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings')))
+EOL
+`
+    for param_source in params_sources
+    do
+        local _dot_setting=$(echo "$param_source" | cut -d= -f1)
+        local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)"
+        local short_setting=$(echo "$_dot_setting" | cut -d. -f3)
+        local _getter="get__${short_setting}"
+        local source="$(echo $param_source | cut -d= -f2)"
+
+        # Get value from getter if exists
+        if type $getter | grep -q '^function$' 2>/dev/null; then
+            old[$short_setting]="$($getter)"
+        
+        # Get value from app settings
+        elif [[ "$source" == "settings" ]] ; then
+            old[$short_setting]="$(ynh_app_setting_get $app $short_setting)"
+        
+        # Get value from a kind of key/value file
+        elif [[ "$source" == *":"* ]] ; then
+            local source_key="$(echo "$source" | cut -d: -f1)"
+            source_key=${source_key:-$short_setting}
+            local source_file="$(echo "$source" | cut -d: -f2)"
+            old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")"
+        
+        # Specific case for files (all content of the file is the source)
+        else
+            old[$short_setting]="$source"
+        fi
+
+    done
+    
+    
+}
+
+_ynh_panel_apply() {
+    for short_setting in "${!dot_settings[@]}"
+    do
+        local setter="set__${short_setting}"
+        local source="$sources[$short_setting]"
+        
+        # Apply setter if exists
+        if type $setter | grep -q '^function$' 2>/dev/null; then
+            $setter
+
+        # Copy file in right place 
+        elif [[ "$source" == "settings" ]] ; then
+            ynh_app_setting_set $app $short_setting "$new[$short_setting]"
+        
+        # Get value from a kind of key/value file
+        elif [[ "$source" == *":"* ]]
+        then
+            local source_key="$(echo "$source" | cut -d: -f1)"
+            source_key=${source_key:-$short_setting}
+            local source_file="$(echo "$source" | cut -d: -f2)"
+            ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]"
+
+        # Specific case for files (all content of the file is the source)
+        else
+            cp "$new[$short_setting]" "$source"
+        fi
+    done
+}
+
+_ynh_panel_show() {
+    for short_setting in "${!old[@]}"
+    do
+        local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
+	    ynh_return "$key=${old[$short_setting]}"
+    done
+}
+
+_ynh_panel_validate() {
+    # Change detection
+    local is_error=true
+    #for changed_status in "${!changed[@]}"
+    for short_setting in "${!dot_settings[@]}"
+    do
+        #TODO file hash
+                file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
+                file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
+                if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
+                then
+                    changed[$setting]=true
+                fi
+        if [[ "$new[$short_setting]" == "$old[$short_setting]" ]]
+        then
+            changed[$short_setting]=false
+        else
+            changed[$short_setting]=true
+            is_error=false
+        fi
+    done
+    
+    # Run validation if something is changed
+    if [[ "$is_error" == "false" ]]
+    then
+
+        for short_setting in "${!dot_settings[@]}"
+        do
+            local result="$(validate__$short_setting)"
+            local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
+            if [ -n "$result" ]
+            then
+                ynh_return "$key=$result"
+                is_error=true
+            fi
+        done
+    fi
+
+    if [[ "$is_error" == "true" ]]
+    then
+        ynh_die
+    fi 
+    
+}
+
+ynh_panel_get() {
+    _ynh_panel_get
+}
+
+ynh_panel_init() {
+    declare -A old=()
+    declare -A changed=()
+    declare -A file_hash=()
+    
+    ynh_panel_get
+}
+
+ynh_panel_show() {
+    _ynh_panel_show
+}
+
+ynh_panel_validate() {
+    _ynh_panel_validate
+}
+
+ynh_panel_apply() {
+    _ynh_panel_apply
+}
+

From 5fec35ccea72b9073ef75b18570c8df0e38aff7b Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 1 Jun 2021 01:29:26 +0200
Subject: [PATCH 06/51] [fix] No validate function in config panel

---
 data/helpers.d/configpanel | 41 ++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index f648826e4..83130cfe6 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -123,7 +123,7 @@ EOL
         local _dot_setting=$(echo "$param_source" | cut -d= -f1)
         local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)"
         local short_setting=$(echo "$_dot_setting" | cut -d. -f3)
-        local _getter="get__${short_setting}"
+        local getter="get__${short_setting}"
         local source="$(echo $param_source | cut -d= -f2)"
 
         # Get value from getter if exists
@@ -195,12 +195,12 @@ _ynh_panel_validate() {
     for short_setting in "${!dot_settings[@]}"
     do
         #TODO file hash
-                file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
-                file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
-                if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
-                then
-                    changed[$setting]=true
-                fi
+        file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
+        file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
+        if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
+        then
+            changed[$setting]=true
+        fi
         if [[ "$new[$short_setting]" == "$old[$short_setting]" ]]
         then
             changed[$short_setting]=false
@@ -216,10 +216,13 @@ _ynh_panel_validate() {
 
         for short_setting in "${!dot_settings[@]}"
         do
-            local result="$(validate__$short_setting)"
-            local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
+            local result=""
+            if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then
+                result="$(validate__$short_setting)"
+            fi
             if [ -n "$result" ]
             then
+                local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
                 ynh_return "$key=$result"
                 is_error=true
             fi
@@ -237,14 +240,6 @@ ynh_panel_get() {
     _ynh_panel_get
 }
 
-ynh_panel_init() {
-    declare -A old=()
-    declare -A changed=()
-    declare -A file_hash=()
-    
-    ynh_panel_get
-}
-
 ynh_panel_show() {
     _ynh_panel_show
 }
@@ -257,3 +252,15 @@ ynh_panel_apply() {
     _ynh_panel_apply
 }
 
+ynh_panel_run() {
+    declare -A old=()
+    declare -A changed=()
+    declare -A file_hash=()
+    
+    ynh_panel_get
+    case $1 in
+        show) ynh_panel_show;;
+        apply) ynh_panel_validate && ynh_panel_apply;;
+    esac
+}
+

From 619b26f73c2356b5d256909c84142c64a8b06020 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 13 Aug 2021 13:38:06 +0200
Subject: [PATCH 07/51] [fix] tons of things

---
 data/helpers.d/configpanel | 110 ++++++++++++++++++++++---------------
 1 file changed, 65 insertions(+), 45 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 83130cfe6..5ab199aea 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -99,13 +99,13 @@ ynh_value_set() {
 }
 
 _ynh_panel_get() {
-    
+    set +x 
     # From settings
     local params_sources
     params_sources=`python3 << EOL
 import toml
 from collections import OrderedDict
-with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f:
+with open("../config_panel.toml", "r") as f:
     file_content = f.read()
 loaded_toml = toml.loads(file_content, _dict=OrderedDict)
 
@@ -114,20 +114,23 @@ for panel_name,panel in loaded_toml.items():
         for section_name, section in panel.items():
             if isinstance(section, dict):
                 for name, param in section.items():
-                    if isinstance(param, dict) and param.get('source', '') == 'settings':
+                    if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']:
                         print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings')))
 EOL
 `
-    for param_source in params_sources
+    for param_source in $params_sources
     do
         local _dot_setting=$(echo "$param_source" | cut -d= -f1)
-        local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)"
+        local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $_dot_setting)"
         local short_setting=$(echo "$_dot_setting" | cut -d. -f3)
         local getter="get__${short_setting}"
         local source="$(echo $param_source | cut -d= -f2)"
+        sources[${short_setting}]="$source"
+        file_hash[${short_setting}]=""
+        dot_settings[${short_setting}]="${_dot_setting}"
 
         # Get value from getter if exists
-        if type $getter | grep -q '^function$' 2>/dev/null; then
+        if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
             old[$short_setting]="$($getter)"
         
         # Get value from app settings
@@ -144,38 +147,43 @@ EOL
         # Specific case for files (all content of the file is the source)
         else
             old[$short_setting]="$source"
+            file_hash[$short_setting]="true"
         fi
-
+        set +u 
+        new[$short_setting]="${!_snake_setting}"
+        set -u
     done
+    set -x
     
     
 }
 
 _ynh_panel_apply() {
-    for short_setting in "${!dot_settings[@]}"
+    for short_setting in "${!old[@]}"
     do
         local setter="set__${short_setting}"
-        local source="$sources[$short_setting]"
-        
-        # Apply setter if exists
-        if type $setter | grep -q '^function$' 2>/dev/null; then
-            $setter
+        local source="${sources[$short_setting]}"
+        if [ "${changed[$short_setting]}" == "true" ] ; then 
+            # Apply setter if exists
+            if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+                $setter
 
-        # Copy file in right place 
-        elif [[ "$source" == "settings" ]] ; then
-            ynh_app_setting_set $app $short_setting "$new[$short_setting]"
-        
-        # Get value from a kind of key/value file
-        elif [[ "$source" == *":"* ]]
-        then
-            local source_key="$(echo "$source" | cut -d: -f1)"
-            source_key=${source_key:-$short_setting}
-            local source_file="$(echo "$source" | cut -d: -f2)"
-            ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]"
+            # Copy file in right place 
+            elif [[ "$source" == "settings" ]] ; then
+                ynh_app_setting_set $app $short_setting "${new[$short_setting]}"
+            
+            # Get value from a kind of key/value file
+            elif [[ "$source" == *":"* ]]
+            then
+                local source_key="$(echo "$source" | cut -d: -f1)"
+                source_key=${source_key:-$short_setting}
+                local source_file="$(echo "$source" | cut -d: -f2)"
+                ynh_value_set --file="${source_file}" --key="${source_key}" --value="${new[$short_setting]}"
 
-        # Specific case for files (all content of the file is the source)
-        else
-            cp "$new[$short_setting]" "$source"
+            # Specific case for files (all content of the file is the source)
+            else
+                cp "${new[$short_setting]}" "$source"
+            fi
         fi
     done
 }
@@ -189,24 +197,32 @@ _ynh_panel_show() {
 }
 
 _ynh_panel_validate() {
+    set +x
     # Change detection
     local is_error=true
     #for changed_status in "${!changed[@]}"
-    for short_setting in "${!dot_settings[@]}"
+    for short_setting in "${!old[@]}"
     do
-        #TODO file hash
-        file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
-        file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
-        if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
-        then
-            changed[$setting]=true
-        fi
-        if [[ "$new[$short_setting]" == "$old[$short_setting]" ]]
-        then
-            changed[$short_setting]=false
+        changed[$short_setting]=false
+        if [ ! -z "${file_hash[${short_setting}]}" ] ; then
+            file_hash[old__$short_setting]=""
+            file_hash[new__$short_setting]=""
+            if [ -f "${old[$short_setting]}" ] ; then
+                file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
+            fi
+            if [ -f "${new[$short_setting]}" ] ; then
+                file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1)
+                if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
+                then
+                    changed[$short_setting]=true
+                fi
+            fi
         else
-            changed[$short_setting]=true
-            is_error=false
+            if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]]
+            then
+                changed[$short_setting]=true
+                is_error=false
+            fi
         fi
     done
     
@@ -214,10 +230,10 @@ _ynh_panel_validate() {
     if [[ "$is_error" == "false" ]]
     then
 
-        for short_setting in "${!dot_settings[@]}"
+        for short_setting in "${!old[@]}"
         do
             local result=""
-            if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then
+            if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then
                 result="$(validate__$short_setting)"
             fi
             if [ -n "$result" ]
@@ -233,6 +249,7 @@ _ynh_panel_validate() {
     then
         ynh_die
     fi 
+    set -x
     
 }
 
@@ -253,9 +270,12 @@ ynh_panel_apply() {
 }
 
 ynh_panel_run() {
-    declare -A old=()
-    declare -A changed=()
-    declare -A file_hash=()
+    declare -Ag old=()
+    declare -Ag new=()
+    declare -Ag changed=()
+    declare -Ag file_hash=()
+    declare -Ag sources=()
+    declare -Ag dot_settings=()
     
     ynh_panel_get
     case $1 in

From 596d05ae81d24712c87ec0de72f8deb8248bca9a Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Sat, 14 Aug 2021 14:38:45 +0200
Subject: [PATCH 08/51] [fix] Missing name or bad format management

---
 src/yunohost/app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 49033d8b4..cf218823f 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -2145,7 +2145,7 @@ def _get_app_config_panel(app_id):
         for key, value in panels:
             panel = {
                 "id": key,
-                "name": value["name"],
+                "name": value.get("name", ""),
                 "sections": [],
             }
 
@@ -2158,7 +2158,7 @@ def _get_app_config_panel(app_id):
             for section_key, section_value in sections:
                 section = {
                     "id": section_key,
-                    "name": section_value["name"],
+                    "name": section_value.get("name", ""),
                     "options": [],
                 }
 

From d8cdc20e0ec4f3af992d3e7f22ed1d9218bd38c2 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 11 May 2020 19:13:10 +0200
Subject: [PATCH 09/51] [wip] Allow file upload from config-panel

---
 src/yunohost/app.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index a48400a8e..a8cd6aa40 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -33,6 +33,7 @@ import re
 import subprocess
 import glob
 import urllib.parse
+import base64
 import tempfile
 from collections import OrderedDict
 
@@ -1874,6 +1875,7 @@ def app_config_apply(operation_logger, app, args):
     }
     args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
 
+    upload_dir = None
     for tab in config_panel.get("panel", []):
         tab_id = tab["id"]  # this makes things easier to debug on crash
         for section in tab.get("sections", []):
@@ -1885,6 +1887,23 @@ def app_config_apply(operation_logger, app, args):
                 ).upper()
 
                 if generated_name in args:
+                    # Upload files from API
+                    # A file arg contains a string with "FILENAME:BASE64_CONTENT"
+                    if option["type"] == "file" and msettings.get('interface') == 'api':
+                        if upload_dir is None:
+                            upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
+                        filename, args[generated_name] = args[generated_name].split(':')
+                        logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
+                        file_path = os.join(upload_dir, filename)
+                        try:
+                            with open(file_path, 'wb') as f:
+                                f.write(args[generated_name])
+                        except IOError as e:
+                            raise YunohostError("cannot_write_file", file=file_path, error=str(e))
+                        except Exception as e:
+                            raise YunohostError("error_writing_file", file=file_path, error=str(e))
+                        args[generated_name] = file_path
+
                     logger.debug(
                         "include into env %s=%s", generated_name, args[generated_name]
                     )
@@ -1906,6 +1925,11 @@ def app_config_apply(operation_logger, app, args):
         env=env,
     )[0]
 
+    # Delete files uploaded from API
+    if msettings.get('interface') == 'api':
+        if upload_dir is not None:
+            shutil.rmtree(upload_dir)
+
     if return_code != 0:
         msg = (
             "'script/config apply' return value code: %s (considered as an error)"

From fb0d23533e2712f4c08e09b9febae1ced8877459 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 8 Jun 2020 19:18:22 +0200
Subject: [PATCH 10/51] [fix] Several files with same name

---
 src/yunohost/app.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index a8cd6aa40..1a0ee9087 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1889,15 +1889,21 @@ def app_config_apply(operation_logger, app, args):
                 if generated_name in args:
                     # Upload files from API
                     # A file arg contains a string with "FILENAME:BASE64_CONTENT"
-                    if option["type"] == "file" and msettings.get('interface') == 'api':
+                    if 'type' in option and option["type"] == "file" \
+                       and msettings.get('interface') == 'api':
                         if upload_dir is None:
                             upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
-                        filename, args[generated_name] = args[generated_name].split(':')
+                        filename = args[generated_name + '[name]']
+                        content = args[generated_name]
                         logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
-                        file_path = os.join(upload_dir, filename)
+                        file_path = os.path.join(upload_dir, filename)
+                        i = 2
+                        while os.path.exists(file_path):
+                            file_path = os.path.join(upload_dir, filename + (".%d" % i))
+                            i += 1
                         try:
                             with open(file_path, 'wb') as f:
-                                f.write(args[generated_name])
+                                f.write(content.decode("base64"))
                         except IOError as e:
                             raise YunohostError("cannot_write_file", file=file_path, error=str(e))
                         except Exception as e:
@@ -1914,7 +1920,7 @@ def app_config_apply(operation_logger, app, args):
     # for debug purpose
     for key in args:
         if key not in env:
-            logger.warning(
+            logger.debug(
                 "Ignore key '%s' from arguments because it is not in the config", key
             )
 

From 975bf4edcbdeabd2f9487dca2f9e56926eb1f64b Mon Sep 17 00:00:00 2001
From: "ljf (zamentur)" <zamentur@users.noreply.github.com>
Date: Wed, 7 Oct 2020 00:31:20 +0200
Subject: [PATCH 11/51] [enh] Replace os.path.join to improve security

---
 src/yunohost/app.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 1a0ee9087..324159859 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1896,10 +1896,14 @@ def app_config_apply(operation_logger, app, args):
                         filename = args[generated_name + '[name]']
                         content = args[generated_name]
                         logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
-                        file_path = os.path.join(upload_dir, filename)
+                        
+                        # Filename is given by user of the API. For security reason, we have replaced 
+                        # os.path.join to avoid the user to be able to rewrite a file in filesystem
+                        # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
+                        file_path = os.path.normpath(upload_dir + "/" + filename)
                         i = 2
                         while os.path.exists(file_path):
-                            file_path = os.path.join(upload_dir, filename + (".%d" % i))
+                            file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
                             i += 1
                         try:
                             with open(file_path, 'wb') as f:

From 284554598521af8305f521b0eeabf2a42f95167e Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 31 May 2021 16:32:19 +0200
Subject: [PATCH 12/51] [fix] Base64 python3 change

---
 src/yunohost/app.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 324159859..4006c1ec4 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1852,7 +1852,7 @@ def app_config_apply(operation_logger, app, args):
     logger.warning(m18n.n("experimental_feature"))
 
     from yunohost.hook import hook_exec
-
+    from base64 import b64decode
     installed = _is_installed(app)
     if not installed:
         raise YunohostValidationError(
@@ -1896,8 +1896,8 @@ def app_config_apply(operation_logger, app, args):
                         filename = args[generated_name + '[name]']
                         content = args[generated_name]
                         logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
-                        
-                        # Filename is given by user of the API. For security reason, we have replaced 
+
+                        # Filename is given by user of the API. For security reason, we have replaced
                         # os.path.join to avoid the user to be able to rewrite a file in filesystem
                         # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
                         file_path = os.path.normpath(upload_dir + "/" + filename)
@@ -1907,7 +1907,7 @@ def app_config_apply(operation_logger, app, args):
                             i += 1
                         try:
                             with open(file_path, 'wb') as f:
-                                f.write(content.decode("base64"))
+                                f.write(b64decode(content))
                         except IOError as e:
                             raise YunohostError("cannot_write_file", file=file_path, error=str(e))
                         except Exception as e:

From 1c636fe1ca8b4838ad99e8f69358e8b8e9e4e4c7 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 1 Jun 2021 00:41:37 +0200
Subject: [PATCH 13/51] [enh] Add configpanel helpers

---
 data/helpers.d/configpanel | 259 +++++++++++++++++++++++++++++++++++++
 1 file changed, 259 insertions(+)
 create mode 100644 data/helpers.d/configpanel

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
new file mode 100644
index 000000000..f648826e4
--- /dev/null
+++ b/data/helpers.d/configpanel
@@ -0,0 +1,259 @@
+#!/bin/bash
+
+ynh_lowerdot_to_uppersnake() {
+    local lowerdot
+    lowerdot=$(echo "$1" | cut -d= -f1 | sed "s/\./_/g")
+    echo "${lowerdot^^}"
+}
+
+# Get a value from heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_value_get --file=PATH --key=KEY
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to get
+#
+# This helpers match several var affectation use case in several languages
+# We don't use jq or equivalent to keep comments and blank space in files
+# This helpers work line by line, it is not able to work correctly 
+# if you have several identical keys in your files
+# 
+# Example of line this helpers can managed correctly
+# .yml
+#     title: YunoHost documentation
+#     email: 'yunohost@yunohost.org'
+# .json
+#     "theme": "colib'ris",
+#     "port": 8102
+#     "some_boolean":     false,
+#     "user": null
+# .ini
+#     some_boolean = On
+#     action = "Clear"
+#     port = 20
+# .php
+#     $user=
+#     user => 20
+# .py
+#     USER = 8102
+#     user = 'https://donate.local'
+#     CUSTOM['user'] = 'YunoHost'
+# Requires YunoHost version 4.3 or higher.
+ynh_value_get() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fk
+    local -A args_array=( [f]=file= [k]=key= )
+    local file
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    
+    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    
+    local first_char="${crazy_value:0:1}"
+    if [[ "$first_char" == '"' ]] ; then
+        echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1  | sed 's/\\"/"/g'
+    elif [[ "$first_char" == "'" ]] ; then
+        echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1  | sed "s/\\\\'/'/g"
+    else
+        echo "$crazy_value"
+    fi
+}
+
+# Set a value into heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_value_set --file=PATH --key=KEY --value=VALUE
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to set
+# | arg: -v, --value=     - the value to set
+#
+# Requires YunoHost version 4.3 or higher.
+ynh_value_set() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fkv
+    local -A args_array=( [f]=file= [k]=key= [v]=value=)
+    local file
+    local key
+    local value
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    
+    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    local first_char="${crazy_value:0:1}"
+    if [[ "$first_char" == '"' ]] ; then
+        value="$(echo "$value" | sed 's/"/\\"/g')"
+        sed -ri "s%^(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+    elif [[ "$first_char" == "'" ]] ; then
+        value="$(echo "$value" | sed "s/'/\\\\'/g")"
+        sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+    else
+        if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
+            value="\"$(echo "$value" | sed 's/"/\\"/g')\""
+        fi
+        sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+    fi
+}
+
+_ynh_panel_get() {
+    
+    # From settings
+    local params_sources
+    params_sources=`python3 << EOL
+import toml
+from collections import OrderedDict
+with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f:
+    file_content = f.read()
+loaded_toml = toml.loads(file_content, _dict=OrderedDict)
+
+for panel_name,panel in loaded_toml.items():
+    if isinstance(panel, dict):
+        for section_name, section in panel.items():
+            if isinstance(section, dict):
+                for name, param in section.items():
+                    if isinstance(param, dict) and param.get('source', '') == 'settings':
+                        print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings')))
+EOL
+`
+    for param_source in params_sources
+    do
+        local _dot_setting=$(echo "$param_source" | cut -d= -f1)
+        local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)"
+        local short_setting=$(echo "$_dot_setting" | cut -d. -f3)
+        local _getter="get__${short_setting}"
+        local source="$(echo $param_source | cut -d= -f2)"
+
+        # Get value from getter if exists
+        if type $getter | grep -q '^function$' 2>/dev/null; then
+            old[$short_setting]="$($getter)"
+        
+        # Get value from app settings
+        elif [[ "$source" == "settings" ]] ; then
+            old[$short_setting]="$(ynh_app_setting_get $app $short_setting)"
+        
+        # Get value from a kind of key/value file
+        elif [[ "$source" == *":"* ]] ; then
+            local source_key="$(echo "$source" | cut -d: -f1)"
+            source_key=${source_key:-$short_setting}
+            local source_file="$(echo "$source" | cut -d: -f2)"
+            old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")"
+        
+        # Specific case for files (all content of the file is the source)
+        else
+            old[$short_setting]="$source"
+        fi
+
+    done
+    
+    
+}
+
+_ynh_panel_apply() {
+    for short_setting in "${!dot_settings[@]}"
+    do
+        local setter="set__${short_setting}"
+        local source="$sources[$short_setting]"
+        
+        # Apply setter if exists
+        if type $setter | grep -q '^function$' 2>/dev/null; then
+            $setter
+
+        # Copy file in right place 
+        elif [[ "$source" == "settings" ]] ; then
+            ynh_app_setting_set $app $short_setting "$new[$short_setting]"
+        
+        # Get value from a kind of key/value file
+        elif [[ "$source" == *":"* ]]
+        then
+            local source_key="$(echo "$source" | cut -d: -f1)"
+            source_key=${source_key:-$short_setting}
+            local source_file="$(echo "$source" | cut -d: -f2)"
+            ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]"
+
+        # Specific case for files (all content of the file is the source)
+        else
+            cp "$new[$short_setting]" "$source"
+        fi
+    done
+}
+
+_ynh_panel_show() {
+    for short_setting in "${!old[@]}"
+    do
+        local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
+	    ynh_return "$key=${old[$short_setting]}"
+    done
+}
+
+_ynh_panel_validate() {
+    # Change detection
+    local is_error=true
+    #for changed_status in "${!changed[@]}"
+    for short_setting in "${!dot_settings[@]}"
+    do
+        #TODO file hash
+                file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
+                file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
+                if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
+                then
+                    changed[$setting]=true
+                fi
+        if [[ "$new[$short_setting]" == "$old[$short_setting]" ]]
+        then
+            changed[$short_setting]=false
+        else
+            changed[$short_setting]=true
+            is_error=false
+        fi
+    done
+    
+    # Run validation if something is changed
+    if [[ "$is_error" == "false" ]]
+    then
+
+        for short_setting in "${!dot_settings[@]}"
+        do
+            local result="$(validate__$short_setting)"
+            local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
+            if [ -n "$result" ]
+            then
+                ynh_return "$key=$result"
+                is_error=true
+            fi
+        done
+    fi
+
+    if [[ "$is_error" == "true" ]]
+    then
+        ynh_die
+    fi 
+    
+}
+
+ynh_panel_get() {
+    _ynh_panel_get
+}
+
+ynh_panel_init() {
+    declare -A old=()
+    declare -A changed=()
+    declare -A file_hash=()
+    
+    ynh_panel_get
+}
+
+ynh_panel_show() {
+    _ynh_panel_show
+}
+
+ynh_panel_validate() {
+    _ynh_panel_validate
+}
+
+ynh_panel_apply() {
+    _ynh_panel_apply
+}
+

From caf2a9d6d13b9c5be0068585d0a3617c2fdc35a7 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 1 Jun 2021 01:29:26 +0200
Subject: [PATCH 14/51] [fix] No validate function in config panel

---
 data/helpers.d/configpanel | 41 ++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index f648826e4..83130cfe6 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -123,7 +123,7 @@ EOL
         local _dot_setting=$(echo "$param_source" | cut -d= -f1)
         local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)"
         local short_setting=$(echo "$_dot_setting" | cut -d. -f3)
-        local _getter="get__${short_setting}"
+        local getter="get__${short_setting}"
         local source="$(echo $param_source | cut -d= -f2)"
 
         # Get value from getter if exists
@@ -195,12 +195,12 @@ _ynh_panel_validate() {
     for short_setting in "${!dot_settings[@]}"
     do
         #TODO file hash
-                file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
-                file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
-                if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
-                then
-                    changed[$setting]=true
-                fi
+        file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
+        file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
+        if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
+        then
+            changed[$setting]=true
+        fi
         if [[ "$new[$short_setting]" == "$old[$short_setting]" ]]
         then
             changed[$short_setting]=false
@@ -216,10 +216,13 @@ _ynh_panel_validate() {
 
         for short_setting in "${!dot_settings[@]}"
         do
-            local result="$(validate__$short_setting)"
-            local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
+            local result=""
+            if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then
+                result="$(validate__$short_setting)"
+            fi
             if [ -n "$result" ]
             then
+                local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
                 ynh_return "$key=$result"
                 is_error=true
             fi
@@ -237,14 +240,6 @@ ynh_panel_get() {
     _ynh_panel_get
 }
 
-ynh_panel_init() {
-    declare -A old=()
-    declare -A changed=()
-    declare -A file_hash=()
-    
-    ynh_panel_get
-}
-
 ynh_panel_show() {
     _ynh_panel_show
 }
@@ -257,3 +252,15 @@ ynh_panel_apply() {
     _ynh_panel_apply
 }
 
+ynh_panel_run() {
+    declare -A old=()
+    declare -A changed=()
+    declare -A file_hash=()
+    
+    ynh_panel_get
+    case $1 in
+        show) ynh_panel_show;;
+        apply) ynh_panel_validate && ynh_panel_apply;;
+    esac
+}
+

From ed0915cf81a42c1ae12b9897cd5b2827c5b96ff6 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 13 Aug 2021 13:38:06 +0200
Subject: [PATCH 15/51] [fix] tons of things

---
 data/helpers.d/configpanel | 110 ++++++++++++++++++++++---------------
 1 file changed, 65 insertions(+), 45 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 83130cfe6..5ab199aea 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -99,13 +99,13 @@ ynh_value_set() {
 }
 
 _ynh_panel_get() {
-    
+    set +x 
     # From settings
     local params_sources
     params_sources=`python3 << EOL
 import toml
 from collections import OrderedDict
-with open("/etc/yunohost/apps/vpnclient/config_panel.toml", "r") as f:
+with open("../config_panel.toml", "r") as f:
     file_content = f.read()
 loaded_toml = toml.loads(file_content, _dict=OrderedDict)
 
@@ -114,20 +114,23 @@ for panel_name,panel in loaded_toml.items():
         for section_name, section in panel.items():
             if isinstance(section, dict):
                 for name, param in section.items():
-                    if isinstance(param, dict) and param.get('source', '') == 'settings':
+                    if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']:
                         print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings')))
 EOL
 `
-    for param_source in params_sources
+    for param_source in $params_sources
     do
         local _dot_setting=$(echo "$param_source" | cut -d= -f1)
-        local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_setting)"
+        local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $_dot_setting)"
         local short_setting=$(echo "$_dot_setting" | cut -d. -f3)
         local getter="get__${short_setting}"
         local source="$(echo $param_source | cut -d= -f2)"
+        sources[${short_setting}]="$source"
+        file_hash[${short_setting}]=""
+        dot_settings[${short_setting}]="${_dot_setting}"
 
         # Get value from getter if exists
-        if type $getter | grep -q '^function$' 2>/dev/null; then
+        if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
             old[$short_setting]="$($getter)"
         
         # Get value from app settings
@@ -144,38 +147,43 @@ EOL
         # Specific case for files (all content of the file is the source)
         else
             old[$short_setting]="$source"
+            file_hash[$short_setting]="true"
         fi
-
+        set +u 
+        new[$short_setting]="${!_snake_setting}"
+        set -u
     done
+    set -x
     
     
 }
 
 _ynh_panel_apply() {
-    for short_setting in "${!dot_settings[@]}"
+    for short_setting in "${!old[@]}"
     do
         local setter="set__${short_setting}"
-        local source="$sources[$short_setting]"
-        
-        # Apply setter if exists
-        if type $setter | grep -q '^function$' 2>/dev/null; then
-            $setter
+        local source="${sources[$short_setting]}"
+        if [ "${changed[$short_setting]}" == "true" ] ; then 
+            # Apply setter if exists
+            if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+                $setter
 
-        # Copy file in right place 
-        elif [[ "$source" == "settings" ]] ; then
-            ynh_app_setting_set $app $short_setting "$new[$short_setting]"
-        
-        # Get value from a kind of key/value file
-        elif [[ "$source" == *":"* ]]
-        then
-            local source_key="$(echo "$source" | cut -d: -f1)"
-            source_key=${source_key:-$short_setting}
-            local source_file="$(echo "$source" | cut -d: -f2)"
-            ynh_value_set --file="${source_file}" --key="${source_key}" --value="$new[$short_setting]"
+            # Copy file in right place 
+            elif [[ "$source" == "settings" ]] ; then
+                ynh_app_setting_set $app $short_setting "${new[$short_setting]}"
+            
+            # Get value from a kind of key/value file
+            elif [[ "$source" == *":"* ]]
+            then
+                local source_key="$(echo "$source" | cut -d: -f1)"
+                source_key=${source_key:-$short_setting}
+                local source_file="$(echo "$source" | cut -d: -f2)"
+                ynh_value_set --file="${source_file}" --key="${source_key}" --value="${new[$short_setting]}"
 
-        # Specific case for files (all content of the file is the source)
-        else
-            cp "$new[$short_setting]" "$source"
+            # Specific case for files (all content of the file is the source)
+            else
+                cp "${new[$short_setting]}" "$source"
+            fi
         fi
     done
 }
@@ -189,24 +197,32 @@ _ynh_panel_show() {
 }
 
 _ynh_panel_validate() {
+    set +x
     # Change detection
     local is_error=true
     #for changed_status in "${!changed[@]}"
-    for short_setting in "${!dot_settings[@]}"
+    for short_setting in "${!old[@]}"
     do
-        #TODO file hash
-        file_hash[$setting]=$(sha256sum "$_source" | cut -d' ' -f1)
-        file_hash[$form_setting]=$(sha256sum "${!form_setting}" | cut -d' ' -f1)
-        if [[ "${file_hash[$setting]}" != "${file_hash[$form_setting]}" ]]
-        then
-            changed[$setting]=true
-        fi
-        if [[ "$new[$short_setting]" == "$old[$short_setting]" ]]
-        then
-            changed[$short_setting]=false
+        changed[$short_setting]=false
+        if [ ! -z "${file_hash[${short_setting}]}" ] ; then
+            file_hash[old__$short_setting]=""
+            file_hash[new__$short_setting]=""
+            if [ -f "${old[$short_setting]}" ] ; then
+                file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
+            fi
+            if [ -f "${new[$short_setting]}" ] ; then
+                file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1)
+                if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
+                then
+                    changed[$short_setting]=true
+                fi
+            fi
         else
-            changed[$short_setting]=true
-            is_error=false
+            if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]]
+            then
+                changed[$short_setting]=true
+                is_error=false
+            fi
         fi
     done
     
@@ -214,10 +230,10 @@ _ynh_panel_validate() {
     if [[ "$is_error" == "false" ]]
     then
 
-        for short_setting in "${!dot_settings[@]}"
+        for short_setting in "${!old[@]}"
         do
             local result=""
-            if type validate__$short_setting | grep -q '^function$' 2>/dev/null; then
+            if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then
                 result="$(validate__$short_setting)"
             fi
             if [ -n "$result" ]
@@ -233,6 +249,7 @@ _ynh_panel_validate() {
     then
         ynh_die
     fi 
+    set -x
     
 }
 
@@ -253,9 +270,12 @@ ynh_panel_apply() {
 }
 
 ynh_panel_run() {
-    declare -A old=()
-    declare -A changed=()
-    declare -A file_hash=()
+    declare -Ag old=()
+    declare -Ag new=()
+    declare -Ag changed=()
+    declare -Ag file_hash=()
+    declare -Ag sources=()
+    declare -Ag dot_settings=()
     
     ynh_panel_get
     case $1 in

From 65fc06e3e743ed6bff30ace4ca0800c54e58b253 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Sat, 14 Aug 2021 14:38:45 +0200
Subject: [PATCH 16/51] [fix] Missing name or bad format management

---
 src/yunohost/app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 4006c1ec4..d8e4748f6 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -2152,7 +2152,7 @@ def _get_app_config_panel(app_id):
         for key, value in panels:
             panel = {
                 "id": key,
-                "name": value["name"],
+                "name": value.get("name", ""),
                 "sections": [],
             }
 
@@ -2165,7 +2165,7 @@ def _get_app_config_panel(app_id):
             for section_key, section_value in sections:
                 section = {
                     "id": section_key,
-                    "name": section_value["name"],
+                    "name": section_value.get("name", ""),
                     "options": [],
                 }
 

From fddb79e841470a58f8bea31293417f80c3e1dd7b Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Tue, 17 Aug 2021 17:07:26 +0200
Subject: [PATCH 17/51] [wip] Reduce config panel var

---
 data/helpers.d/configpanel |  18 ++----
 src/yunohost/app.py        | 116 ++++++++++++++++++-------------------
 2 files changed, 60 insertions(+), 74 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 5ab199aea..685f30a98 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -115,19 +115,16 @@ for panel_name,panel in loaded_toml.items():
             if isinstance(section, dict):
                 for name, param in section.items():
                     if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']:
-                        print("%s.%s.%s=%s" %(panel_name, section_name, name, param.get('source', 'settings')))
+                        print("%s=%s" % (name, param.get('source', 'settings')))
 EOL
 `
     for param_source in $params_sources
     do
-        local _dot_setting=$(echo "$param_source" | cut -d= -f1)
-        local _snake_setting="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $_dot_setting)"
-        local short_setting=$(echo "$_dot_setting" | cut -d. -f3)
+        local short_setting="$(echo $param_source | cut -d= -f1)"
         local getter="get__${short_setting}"
         local source="$(echo $param_source | cut -d= -f2)"
         sources[${short_setting}]="$source"
         file_hash[${short_setting}]=""
-        dot_settings[${short_setting}]="${_dot_setting}"
 
         # Get value from getter if exists
         if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
@@ -149,9 +146,6 @@ EOL
             old[$short_setting]="$source"
             file_hash[$short_setting]="true"
         fi
-        set +u 
-        new[$short_setting]="${!_snake_setting}"
-        set -u
     done
     set -x
     
@@ -191,8 +185,7 @@ _ynh_panel_apply() {
 _ynh_panel_show() {
     for short_setting in "${!old[@]}"
     do
-        local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
-	    ynh_return "$key=${old[$short_setting]}"
+	    ynh_return "${short_setting}=${old[$short_setting]}"
     done
 }
 
@@ -238,7 +231,7 @@ _ynh_panel_validate() {
             fi
             if [ -n "$result" ]
             then
-                local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
+                local key="YNH_ERROR_${$short_setting}"
                 ynh_return "$key=$result"
                 is_error=true
             fi
@@ -247,7 +240,7 @@ _ynh_panel_validate() {
 
     if [[ "$is_error" == "true" ]]
     then
-        ynh_die
+        ynh_die ""
     fi 
     set -x
     
@@ -275,7 +268,6 @@ ynh_panel_run() {
     declare -Ag changed=()
     declare -Ag file_hash=()
     declare -Ag sources=()
-    declare -Ag dot_settings=()
     
     ynh_panel_get
     case $1 in
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index d8e4748f6..faa5098c9 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1782,59 +1782,49 @@ def app_config_show_panel(operation_logger, app):
         }
 
     env = {
-        "YNH_APP_ID": app_id,
-        "YNH_APP_INSTANCE_NAME": app,
-        "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
+        "app_id": app_id,
+        "app": app,
+        "app_instance_nb": str(app_instance_nb),
     }
 
-    # FIXME: this should probably be ran in a tmp workdir...
-    return_code, parsed_values = hook_exec(
-        config_script, args=["show"], env=env, return_format="plain_dict"
-    )
-
-    if return_code != 0:
-        raise Exception(
-            "script/config show return value code: %s (considered as an error)",
-            return_code,
+    try:
+        ret, parsed_values = hook_exec(
+            config_script, args=["show"], env=env, return_format="plain_dict"
         )
+    # Here again, calling hook_exec could fail miserably, or get
+    # manually interrupted (by mistake or because script was stuck)
+    except (KeyboardInterrupt, EOFError, Exception):
+        raise YunohostError("unexpected_error")
 
     logger.debug("Generating global variables:")
     for tab in config_panel.get("panel", []):
-        tab_id = tab["id"]  # this makes things easier to debug on crash
         for section in tab.get("sections", []):
-            section_id = section["id"]
             for option in section.get("options", []):
-                option_name = option["name"]
-                generated_name = (
-                    "YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)
-                ).upper()
-                option["name"] = generated_name
                 logger.debug(
-                    " * '%s'.'%s'.'%s' -> %s",
+                    " * '%s'.'%s'.'%s'",
                     tab.get("name"),
                     section.get("name"),
                     option.get("name"),
-                    generated_name,
                 )
 
-                if generated_name in parsed_values:
+                if option['name'] in parsed_values:
                     # code is not adapted for that so we have to mock expected format :/
                     if option.get("type") == "boolean":
-                        if parsed_values[generated_name].lower() in ("true", "1", "y"):
-                            option["default"] = parsed_values[generated_name]
+                        if parsed_values[option['name']].lower() in ("true", "1", "y"):
+                            option["default"] = parsed_values[option['name']]
                         else:
                             del option["default"]
                     else:
-                        option["default"] = parsed_values[generated_name]
+                        option["default"] = parsed_values[option['name']]
 
                     args_dict = _parse_args_in_yunohost_format(
-                        {option["name"]: parsed_values[generated_name]}, [option]
+                        parsed_values, [option]
                     )
                     option["default"] = args_dict[option["name"]][0]
                 else:
                     logger.debug(
                         "Variable '%s' is not declared by config script, using default",
-                        generated_name,
+                        option['name'],
                     )
                     # do nothing, we'll use the default if present
 
@@ -1869,32 +1859,26 @@ def app_config_apply(operation_logger, app, args):
     operation_logger.start()
     app_id, app_instance_nb = _parse_app_instance_name(app)
     env = {
-        "YNH_APP_ID": app_id,
-        "YNH_APP_INSTANCE_NAME": app,
-        "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
+        "app_id": app_id,
+        "app": app,
+        "app_instance_nb": str(app_instance_nb),
     }
     args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
 
     upload_dir = None
     for tab in config_panel.get("panel", []):
-        tab_id = tab["id"]  # this makes things easier to debug on crash
         for section in tab.get("sections", []):
-            section_id = section["id"]
             for option in section.get("options", []):
-                option_name = option["name"]
-                generated_name = (
-                    "YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)
-                ).upper()
 
-                if generated_name in args:
+                if option['name'] in args:
                     # Upload files from API
                     # A file arg contains a string with "FILENAME:BASE64_CONTENT"
                     if 'type' in option and option["type"] == "file" \
                        and msettings.get('interface') == 'api':
                         if upload_dir is None:
                             upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
-                        filename = args[generated_name + '[name]']
-                        content = args[generated_name]
+                        filename = args[option['name'] + '[name]']
+                        content = args[option['name']]
                         logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
 
                         # Filename is given by user of the API. For security reason, we have replaced
@@ -1912,14 +1896,14 @@ def app_config_apply(operation_logger, app, args):
                             raise YunohostError("cannot_write_file", file=file_path, error=str(e))
                         except Exception as e:
                             raise YunohostError("error_writing_file", file=file_path, error=str(e))
-                        args[generated_name] = file_path
+                        args[option['name']] = file_path
 
                     logger.debug(
-                        "include into env %s=%s", generated_name, args[generated_name]
+                        "include into env %s=%s", option['name'], args[option['name']]
                     )
-                    env[generated_name] = args[generated_name]
+                    env[option['name']] = args[option['name']]
                 else:
-                    logger.debug("no value for key id %s", generated_name)
+                    logger.debug("no value for key id %s", option['name'])
 
     # for debug purpose
     for key in args:
@@ -1928,25 +1912,21 @@ def app_config_apply(operation_logger, app, args):
                 "Ignore key '%s' from arguments because it is not in the config", key
             )
 
-    # FIXME: this should probably be ran in a tmp workdir...
-    return_code = hook_exec(
-        config_script,
-        args=["apply"],
-        env=env,
-    )[0]
-
-    # Delete files uploaded from API
-    if msettings.get('interface') == 'api':
-        if upload_dir is not None:
-            shutil.rmtree(upload_dir)
-
-    if return_code != 0:
-        msg = (
-            "'script/config apply' return value code: %s (considered as an error)"
-            % return_code
+    try:
+        hook_exec(
+            config_script,
+            args=["apply"],
+            env=env
         )
-        operation_logger.error(msg)
-        raise Exception(msg)
+    # Here again, calling hook_exec could fail miserably, or get
+    # manually interrupted (by mistake or because script was stuck)
+    except (KeyboardInterrupt, EOFError, Exception):
+        raise YunohostError("unexpected_error")
+    finally:
+        # Delete files uploaded from API
+        if msettings.get('interface') == 'api':
+            if upload_dir is not None:
+                shutil.rmtree(upload_dir)
 
     logger.success("Config updated as expected")
     return {
@@ -2991,16 +2971,30 @@ class DisplayTextArgumentParser(YunoHostArgumentFormatParser):
     def parse(self, question, user_answers):
         print(question["ask"])
 
+class FileArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "file"
+
+
 
 ARGUMENTS_TYPE_PARSERS = {
     "string": StringArgumentParser,
+    "text": StringArgumentParser,
+    "select": StringArgumentParser,
+    "tags": StringArgumentParser,
+    "email": StringArgumentParser,
+    "url": StringArgumentParser,
+    "date": StringArgumentParser,
+    "time": StringArgumentParser,
+    "color": StringArgumentParser,
     "password": PasswordArgumentParser,
     "path": PathArgumentParser,
     "boolean": BooleanArgumentParser,
     "domain": DomainArgumentParser,
     "user": UserArgumentParser,
     "number": NumberArgumentParser,
+    "range": NumberArgumentParser,
     "display_text": DisplayTextArgumentParser,
+    "file": FileArgumentParser,
 }
 
 

From a89dd4827c1f072b0e808b6b4b669909aad58179 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 20 Aug 2021 16:35:02 +0200
Subject: [PATCH 18/51] [enh] Use yaml for reading hook output

---
 src/yunohost/hook.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py
index 33f5885e2..4d497de76 100644
--- a/src/yunohost/hook.py
+++ b/src/yunohost/hook.py
@@ -34,7 +34,7 @@ from importlib import import_module
 from moulinette import m18n, msettings
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from moulinette.utils import log
-from moulinette.utils.filesystem import read_json
+from moulinette.utils.filesystem import read_yaml
 
 HOOK_FOLDER = "/usr/share/yunohost/hooks/"
 CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/"
@@ -326,7 +326,7 @@ def hook_exec(
     chdir=None,
     env=None,
     user="root",
-    return_format="json",
+    return_format="yaml",
 ):
     """
     Execute hook from a file with arguments
@@ -447,10 +447,10 @@ def _hook_exec_bash(path, args, chdir, env, user, return_format, loggers):
             raw_content = f.read()
         returncontent = {}
 
-        if return_format == "json":
+        if return_format == "yaml":
             if raw_content != "":
                 try:
-                    returncontent = read_json(stdreturn)
+                    returncontent = read_yaml(stdreturn)
                 except Exception as e:
                     raise YunohostError(
                         "hook_json_return_error",

From 98ca514f8fdff264eda071dbeb5244e8548e1657 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 20 Aug 2021 16:35:51 +0200
Subject: [PATCH 19/51] [enh] Rewrite config show, get, set actions

---
 data/actionsmap/yunohost.yml |  45 +++-
 data/helpers.d/configpanel   |  29 +--
 locales/en.json              |   8 +-
 src/yunohost/app.py          | 444 ++++++++++++++++++++++-------------
 4 files changed, 327 insertions(+), 199 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 5df1c0877..d9e3a50d0 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -831,24 +831,47 @@ app:
           subcategory_help: Applications configuration panel
           actions:
 
-              ### app_config_show_panel()
-              show-panel:
+              ### app_config_show()
+              show:
                   action_help: show config panel for the application
                   api: GET /apps/<app>/config-panel
                   arguments:
-                      app:
-                          help: App name
+                    app:
+                        help: App name
+                    panel:
+                        help: Select a specific panel
+                        nargs: '?'
+                    -f:
+                        full: --full
+                        help: Display all info known about the config-panel.
+                        action: store_true
 
-              ### app_config_apply()
-              apply:
+              ### app_config_get()
+              get:
+                  action_help: show config panel for the application
+                  api: GET /apps/<app>/config-panel/<key>
+                  arguments:
+                    app:
+                        help: App name
+                    key:
+                        help: The question identifier 
+
+              ### app_config_set()
+              set:
                   action_help: apply the new configuration
                   api: PUT /apps/<app>/config
                   arguments:
-                      app:
-                          help: App name
-                      -a:
-                          full: --args
-                          help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path")
+                    app:
+                        help: App name
+                    key:
+                        help: The question or panel key 
+                        nargs: '?'
+                    -v:
+                        full: --value
+                        help: new value
+                    -a:
+                        full: --args
+                        help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path")
 
 #############################
 #          Backup           #
diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 685f30a98..5b290629d 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -1,10 +1,5 @@
 #!/bin/bash
 
-ynh_lowerdot_to_uppersnake() {
-    local lowerdot
-    lowerdot=$(echo "$1" | cut -d= -f1 | sed "s/\./_/g")
-    echo "${lowerdot^^}"
-}
 
 # Get a value from heterogeneous file (yaml, json, php, python...)
 #
@@ -99,7 +94,6 @@ ynh_value_set() {
 }
 
 _ynh_panel_get() {
-    set +x 
     # From settings
     local params_sources
     params_sources=`python3 << EOL
@@ -114,7 +108,7 @@ for panel_name,panel in loaded_toml.items():
         for section_name, section in panel.items():
             if isinstance(section, dict):
                 for name, param in section.items():
-                    if isinstance(param, dict) and param.get('type', 'string') not in ['info', 'warning', 'error']:
+                    if isinstance(param, dict) and param.get('type', 'string') not in ['success', 'info', 'warning', 'danger', 'display_text', 'markdown']:
                         print("%s=%s" % (name, param.get('source', 'settings')))
 EOL
 `
@@ -147,7 +141,6 @@ EOL
             file_hash[$short_setting]="true"
         fi
     done
-    set -x
     
     
 }
@@ -164,7 +157,7 @@ _ynh_panel_apply() {
 
             # Copy file in right place 
             elif [[ "$source" == "settings" ]] ; then
-                ynh_app_setting_set $app $short_setting "${new[$short_setting]}"
+                ynh_app_setting_set $app $short_setting "${!short_setting}"
             
             # Get value from a kind of key/value file
             elif [[ "$source" == *":"* ]]
@@ -172,11 +165,11 @@ _ynh_panel_apply() {
                 local source_key="$(echo "$source" | cut -d: -f1)"
                 source_key=${source_key:-$short_setting}
                 local source_file="$(echo "$source" | cut -d: -f2)"
-                ynh_value_set --file="${source_file}" --key="${source_key}" --value="${new[$short_setting]}"
+                ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}"
 
             # Specific case for files (all content of the file is the source)
             else
-                cp "${new[$short_setting]}" "$source"
+                cp "${!short_setting}" "$source"
             fi
         fi
     done
@@ -185,25 +178,25 @@ _ynh_panel_apply() {
 _ynh_panel_show() {
     for short_setting in "${!old[@]}"
     do
-	    ynh_return "${short_setting}=${old[$short_setting]}"
+	    ynh_return "${short_setting}: \"${old[$short_setting]}\""
     done
 }
 
 _ynh_panel_validate() {
-    set +x
     # Change detection
     local is_error=true
     #for changed_status in "${!changed[@]}"
     for short_setting in "${!old[@]}"
     do
         changed[$short_setting]=false
+        [ -z ${!short_setting+x} ] && continue
         if [ ! -z "${file_hash[${short_setting}]}" ] ; then
             file_hash[old__$short_setting]=""
             file_hash[new__$short_setting]=""
             if [ -f "${old[$short_setting]}" ] ; then
                 file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
             fi
-            if [ -f "${new[$short_setting]}" ] ; then
+            if [ -f "${!short_setting}" ] ; then
                 file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1)
                 if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
                 then
@@ -211,7 +204,7 @@ _ynh_panel_validate() {
                 fi
             fi
         else
-            if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]]
+            if [[ "${!short_setting}" != "${old[$short_setting]}" ]]
             then
                 changed[$short_setting]=true
                 is_error=false
@@ -242,7 +235,6 @@ _ynh_panel_validate() {
     then
         ynh_die ""
     fi 
-    set -x
     
 }
 
@@ -264,15 +256,14 @@ ynh_panel_apply() {
 
 ynh_panel_run() {
     declare -Ag old=()
-    declare -Ag new=()
     declare -Ag changed=()
     declare -Ag file_hash=()
     declare -Ag sources=()
     
     ynh_panel_get
     case $1 in
-        show) ynh_panel_show;;
-        apply) ynh_panel_validate && ynh_panel_apply;;
+        show) ynh_panel_get && ynh_panel_show;;
+        apply) ynh_panel_get && ynh_panel_validate && ynh_panel_apply;;
     esac
 }
 
diff --git a/locales/en.json b/locales/en.json
index 693e9d24d..1f13b3e90 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -13,7 +13,7 @@
     "app_already_installed": "{app} is already installed",
     "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.",
     "app_already_up_to_date": "{app} is already up-to-date",
-    "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}'",
+    "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'",
     "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}",
     "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason",
     "app_argument_required": "Argument '{name}' is required",
@@ -143,6 +143,7 @@
     "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system… If you are willing to take that risk anyway, type '{answers}'",
     "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system… If you are willing to take that risk anyway, type '{answers}'",
     "custom_app_url_required": "You must provide a URL to upgrade your custom app {app}",
+    "danger": "Danger:",
     "diagnosis_basesystem_hardware": "Server hardware architecture is {virt} {arch}",
     "diagnosis_basesystem_hardware_model": "Server model is {model}",
     "diagnosis_basesystem_host": "Server is running Debian {debian_version}",
@@ -382,8 +383,9 @@
     "log_app_upgrade": "Upgrade the '{}' app",
     "log_app_makedefault": "Make '{}' the default app",
     "log_app_action_run": "Run action of the '{}' app",
-    "log_app_config_show_panel": "Show the config panel of the '{}' app",
-    "log_app_config_apply": "Apply config to the '{}' app",
+    "log_app_config_show": "Show the config panel of the '{}' app",
+    "log_app_config_get": "Get a specific setting from config panel of the '{}' app",
+    "log_app_config_set": "Apply config to the '{}' app",
     "log_available_on_yunopaste": "This log is now available via {url}",
     "log_backup_create": "Create a backup archive",
     "log_backup_restore_system": "Restore system from a backup archive",
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index faa5098c9..c489cceaa 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -38,6 +38,7 @@ import tempfile
 from collections import OrderedDict
 
 from moulinette import msignals, m18n, msettings
+from moulinette.interfaces.cli import colorize
 from moulinette.core import MoulinetteError
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.network import download_json
@@ -190,10 +191,7 @@ def app_info(app, full=False):
     """
     from yunohost.permission import user_permission_list
 
-    if not _is_installed(app):
-        raise YunohostValidationError(
-            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
-        )
+    _assert_is_installed(app)
 
     local_manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app))
     permissions = user_permission_list(full=True, absolute_urls=True, apps=[app])[
@@ -534,10 +532,8 @@ def app_upgrade(app=[], url=None, file=None, force=False):
     apps = [app_ for i, app_ in enumerate(apps) if app_ not in apps[:i]]
 
     # Abort if any of those app is in fact not installed..
-    for app in [app_ for app_ in apps if not _is_installed(app_)]:
-        raise YunohostValidationError(
-            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
-        )
+    for app_ in apps:
+        _assert_is_installed(app_)
 
     if len(apps) == 0:
         raise YunohostValidationError("apps_already_up_to_date")
@@ -750,7 +746,6 @@ def app_upgrade(app=[], url=None, file=None, force=False):
             for file_to_copy in [
                 "actions.json",
                 "actions.toml",
-                "config_panel.json",
                 "config_panel.toml",
                 "conf",
             ]:
@@ -970,7 +965,6 @@ def app_install(
     for file_to_copy in [
         "actions.json",
         "actions.toml",
-        "config_panel.json",
         "config_panel.toml",
         "conf",
     ]:
@@ -1759,165 +1753,143 @@ def app_action_run(operation_logger, app, action, args=None):
 # * docstrings
 # * merge translations on the json once the workflow is in place
 @is_unit_operation()
-def app_config_show_panel(operation_logger, app):
-    logger.warning(m18n.n("experimental_feature"))
+def app_config_show(operation_logger, app, panel='', full=False):
+    # logger.warning(m18n.n("experimental_feature"))
 
-    from yunohost.hook import hook_exec
-
-    # this will take care of checking if the app is installed
-    app_info_dict = app_info(app)
+    # Check app is installed
+    _assert_is_installed(app)
 
+    panel = panel if panel else ''
     operation_logger.start()
-    config_panel = _get_app_config_panel(app)
-    config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config")
 
-    app_id, app_instance_nb = _parse_app_instance_name(app)
+    # Read config panel toml
+    config_panel = _get_app_config_panel(app, filter_key=panel)
 
-    if not config_panel or not os.path.exists(config_script):
-        return {
-            "app_id": app_id,
-            "app": app,
-            "app_name": app_info_dict["name"],
-            "config_panel": [],
+    if not config_panel:
+        return None
+
+    # Call config script to extract current values
+    parsed_values = _call_config_script(app, 'show')
+
+    # # Check and transform values if needed
+    # options = [option for _, _, option in _get_options_iterator(config_panel)]
+    # args_dict = _parse_args_in_yunohost_format(
+    #     parsed_values, options, False
+    # )
+
+    # Hydrate
+    logger.debug("Hydrating config with current value")
+    for _, _, option in _get_options_iterator(config_panel):
+        if option['name'] in parsed_values:
+            option["value"] = parsed_values[option['name']] #args_dict[option["name"]][0]
+
+    # Format result in full or reduce mode
+    if full:
+        operation_logger.success()
+        return config_panel
+
+    result = OrderedDict()
+    for panel, section, option in _get_options_iterator(config_panel):
+        if panel['id'] not in result:
+            r_panel = result[panel['id']] = OrderedDict()
+        if section['id'] not in r_panel:
+            r_section = r_panel[section['id']] = OrderedDict()
+        r_option = r_section[option['name']] = {
+            "ask": option['ask']['en']
         }
+        if not option.get('optional', False):
+            r_option['ask'] += ' *'
+        if option.get('value', None) is not None:
+            r_option['value'] = option['value']
 
-    env = {
-        "app_id": app_id,
-        "app": app,
-        "app_instance_nb": str(app_instance_nb),
-    }
-
-    try:
-        ret, parsed_values = hook_exec(
-            config_script, args=["show"], env=env, return_format="plain_dict"
-        )
-    # Here again, calling hook_exec could fail miserably, or get
-    # manually interrupted (by mistake or because script was stuck)
-    except (KeyboardInterrupt, EOFError, Exception):
-        raise YunohostError("unexpected_error")
-
-    logger.debug("Generating global variables:")
-    for tab in config_panel.get("panel", []):
-        for section in tab.get("sections", []):
-            for option in section.get("options", []):
-                logger.debug(
-                    " * '%s'.'%s'.'%s'",
-                    tab.get("name"),
-                    section.get("name"),
-                    option.get("name"),
-                )
-
-                if option['name'] in parsed_values:
-                    # code is not adapted for that so we have to mock expected format :/
-                    if option.get("type") == "boolean":
-                        if parsed_values[option['name']].lower() in ("true", "1", "y"):
-                            option["default"] = parsed_values[option['name']]
-                        else:
-                            del option["default"]
-                    else:
-                        option["default"] = parsed_values[option['name']]
-
-                    args_dict = _parse_args_in_yunohost_format(
-                        parsed_values, [option]
-                    )
-                    option["default"] = args_dict[option["name"]][0]
-                else:
-                    logger.debug(
-                        "Variable '%s' is not declared by config script, using default",
-                        option['name'],
-                    )
-                    # do nothing, we'll use the default if present
-
-    return {
-        "app_id": app_id,
-        "app": app,
-        "app_name": app_info_dict["name"],
-        "config_panel": config_panel,
-        "logs": operation_logger.success(),
-    }
+    operation_logger.success()
+    return result
 
 
 @is_unit_operation()
-def app_config_apply(operation_logger, app, args):
-    logger.warning(m18n.n("experimental_feature"))
-
-    from yunohost.hook import hook_exec
-    from base64 import b64decode
-    installed = _is_installed(app)
-    if not installed:
-        raise YunohostValidationError(
-            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
-        )
-
-    config_panel = _get_app_config_panel(app)
-    config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config")
-
-    if not config_panel or not os.path.exists(config_script):
-        # XXX real exception
-        raise Exception("Not config-panel.json nor scripts/config")
+def app_config_get(operation_logger, app, key):
+    # Check app is installed
+    _assert_is_installed(app)
 
     operation_logger.start()
-    app_id, app_instance_nb = _parse_app_instance_name(app)
-    env = {
-        "app_id": app_id,
-        "app": app,
-        "app_instance_nb": str(app_instance_nb),
-    }
-    args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
+
+    # Read config panel toml
+    config_panel = _get_app_config_panel(app, filter_key=key)
+
+    if not config_panel:
+        raise YunohostError("app_config_no_panel")
+
+    # Call config script to extract current values
+    parsed_values = _call_config_script(app, 'show')
+
+    logger.debug("Searching value")
+    short_key = key.split('.')[-1]
+    if short_key not in parsed_values:
+        return None
+
+    return parsed_values[short_key]
+
+    # for panel, section, option in _get_options_iterator(config_panel):
+    #     if option['name'] == short_key:
+    #         # Check and transform values if needed
+    #         args_dict = _parse_args_in_yunohost_format(
+    #             parsed_values, [option], False
+    #         )
+    #         operation_logger.success()
+
+    #         return args_dict[short_key][0]
+
+    # return None
+
+
+@is_unit_operation()
+def app_config_set(operation_logger, app, key=None, value=None, args=None):
+    # Check app is installed
+    _assert_is_installed(app)
+
+    filter_key = key if key else ''
+
+    # Read config panel toml
+    config_panel = _get_app_config_panel(app, filter_key=filter_key)
+
+    if not config_panel:
+        raise YunohostError("app_config_no_panel")
+
+    if args is not None and value is not None:
+        raise YunohostError("app_config_args_value")
+
+    operation_logger.start()
+
+    # Prepare pre answered questions
+    args = {}
+    if args:
+        args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
+    elif value is not None:
+        args = {key: value}
 
     upload_dir = None
-    for tab in config_panel.get("panel", []):
-        for section in tab.get("sections", []):
-            for option in section.get("options", []):
 
-                if option['name'] in args:
-                    # Upload files from API
-                    # A file arg contains a string with "FILENAME:BASE64_CONTENT"
-                    if 'type' in option and option["type"] == "file" \
-                       and msettings.get('interface') == 'api':
-                        if upload_dir is None:
-                            upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
-                        filename = args[option['name'] + '[name]']
-                        content = args[option['name']]
-                        logger.debug("Save uploaded file %s from API into %s", filename, upload_dir)
+    for panel in config_panel.get("panel", []):
 
-                        # Filename is given by user of the API. For security reason, we have replaced
-                        # os.path.join to avoid the user to be able to rewrite a file in filesystem
-                        # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
-                        file_path = os.path.normpath(upload_dir + "/" + filename)
-                        i = 2
-                        while os.path.exists(file_path):
-                            file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
-                            i += 1
-                        try:
-                            with open(file_path, 'wb') as f:
-                                f.write(b64decode(content))
-                        except IOError as e:
-                            raise YunohostError("cannot_write_file", file=file_path, error=str(e))
-                        except Exception as e:
-                            raise YunohostError("error_writing_file", file=file_path, error=str(e))
-                        args[option['name']] = file_path
+        if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
+            msignals.display(colorize("\n" + "=" * 40, 'purple'))
+            msignals.display(colorize(f">>>> {panel['name']}", 'purple'))
+            msignals.display(colorize("=" * 40, 'purple'))
+        for section in panel.get("sections", []):
+            if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
+                msignals.display(colorize(f"\n# {section['name']}", 'purple'))
 
-                    logger.debug(
-                        "include into env %s=%s", option['name'], args[option['name']]
-                    )
-                    env[option['name']] = args[option['name']]
-                else:
-                    logger.debug("no value for key id %s", option['name'])
-
-    # for debug purpose
-    for key in args:
-        if key not in env:
-            logger.debug(
-                "Ignore key '%s' from arguments because it is not in the config", key
+            # Check and ask unanswered questions
+            args_dict = _parse_args_in_yunohost_format(
+                args, section['options']
             )
 
+    # Call config script to extract current values
+    logger.info("Running config script...")
+    env = {key: value[0] for key, value in args_dict.items()}
+
     try:
-        hook_exec(
-            config_script,
-            args=["apply"],
-            env=env
-        )
+        errors = _call_config_script(app, 'apply', env=env)
     # Here again, calling hook_exec could fail miserably, or get
     # manually interrupted (by mistake or because script was stuck)
     except (KeyboardInterrupt, EOFError, Exception):
@@ -1931,10 +1903,51 @@ def app_config_apply(operation_logger, app, args):
     logger.success("Config updated as expected")
     return {
         "app": app,
+        "errors": errors,
         "logs": operation_logger.success(),
     }
 
 
+def _get_options_iterator(config_panel):
+    for panel in config_panel.get("panel", []):
+        for section in panel.get("sections", []):
+            for option in section.get("options", []):
+                yield (panel, section, option)
+
+
+def _call_config_script(app, action, env={}):
+    from yunohost.hook import hook_exec
+
+    # Add default config script if needed
+    config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config")
+    if not os.path.exists(config_script):
+        logger.debug("Adding a default config script")
+        default_script = """#!/bin/bash
+source /usr/share/yunohost/helpers
+ynh_abort_if_errors
+final_path=$(ynh_app_setting_get $app final_path)
+ynh_panel_run $1
+"""
+        write_to_file(config_script, default_script)
+
+    # Call config script to extract current values
+    logger.debug("Calling 'show' action from config script")
+    app_id, app_instance_nb = _parse_app_instance_name(app)
+    env.update({
+        "app_id": app_id,
+        "app": app,
+        "app_instance_nb": str(app_instance_nb),
+    })
+
+    try:
+        _, parsed_values = hook_exec(
+            config_script, args=[action], env=env
+        )
+    except (KeyboardInterrupt, EOFError, Exception):
+        logger.error('Unable to extract some values')
+        parsed_values = {}
+    return parsed_values
+
 def _get_all_installed_apps_id():
     """
     Return something like:
@@ -2036,14 +2049,11 @@ def _get_app_actions(app_id):
     return None
 
 
-def _get_app_config_panel(app_id):
+def _get_app_config_panel(app_id, filter_key=''):
     "Get app config panel stored in json or in toml"
     config_panel_toml_path = os.path.join(
         APPS_SETTING_PATH, app_id, "config_panel.toml"
     )
-    config_panel_json_path = os.path.join(
-        APPS_SETTING_PATH, app_id, "config_panel.json"
-    )
 
     # sample data to get an idea of what is going on
     # this toml extract:
@@ -2121,6 +2131,10 @@ def _get_app_config_panel(app_id):
             "version": toml_config_panel["version"],
             "panel": [],
         }
+        filter_key = filter_key.split('.')
+        filter_panel = filter_key.pop(0)
+        filter_section = filter_key.pop(0) if len(filter_key) > 0 else False
+        filter_option = filter_key.pop(0) if len(filter_key) > 0 else False
 
         panels = [
             key_value
@@ -2130,6 +2144,9 @@ def _get_app_config_panel(app_id):
         ]
 
         for key, value in panels:
+            if filter_panel and key != filter_panel:
+                continue
+
             panel = {
                 "id": key,
                 "name": value.get("name", ""),
@@ -2143,9 +2160,14 @@ def _get_app_config_panel(app_id):
             ]
 
             for section_key, section_value in sections:
+
+                if filter_section and section_key != filter_section:
+                    continue
+
                 section = {
                     "id": section_key,
                     "name": section_value.get("name", ""),
+                    "optional": section_value.get("optional", True),
                     "options": [],
                 }
 
@@ -2156,7 +2178,11 @@ def _get_app_config_panel(app_id):
                 ]
 
                 for option_key, option_value in options:
+                    if filter_option and option_key != filter_option:
+                        continue
+
                     option = dict(option_value)
+                    option["optional"] = option_value.get("optional", section['optional'])
                     option["name"] = option_key
                     option["ask"] = {"en": option["ask"]}
                     if "help" in option:
@@ -2169,9 +2195,6 @@ def _get_app_config_panel(app_id):
 
         return config_panel
 
-    elif os.path.exists(config_panel_json_path):
-        return json.load(open(config_panel_json_path))
-
     return None
 
 
@@ -2615,6 +2638,13 @@ def _is_installed(app):
     return os.path.isdir(APPS_SETTING_PATH + app)
 
 
+def _assert_is_installed(app):
+    if not _is_installed(app):
+        raise YunohostValidationError(
+            "app_not_installed", app=app, all_apps=_get_all_installed_apps_id()
+        )
+
+
 def _installed_apps():
     return os.listdir(APPS_SETTING_PATH)
 
@@ -2727,10 +2757,13 @@ class YunoHostArgumentFormatParser(object):
         parsed_question = Question()
 
         parsed_question.name = question["name"]
+        parsed_question.type = question.get("type", 'string')
         parsed_question.default = question.get("default", None)
         parsed_question.choices = question.get("choices", [])
         parsed_question.optional = question.get("optional", False)
         parsed_question.ask = question.get("ask")
+        parsed_question.help = question.get("help")
+        parsed_question.helpLink = question.get("helpLink")
         parsed_question.value = user_answers.get(parsed_question.name)
 
         if parsed_question.ask is None:
@@ -2742,24 +2775,28 @@ class YunoHostArgumentFormatParser(object):
 
         return parsed_question
 
-    def parse(self, question, user_answers):
+    def parse(self, question, user_answers, check_required=True):
         question = self.parse_question(question, user_answers)
 
-        if question.value is None:
+        if question.value is None and not getattr(self, "readonly", False):
             text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
                 question
             )
-
             try:
                 question.value = msignals.prompt(
-                    text_for_user_input_in_cli, self.hide_user_input_in_prompt
+                    message=text_for_user_input_in_cli,
+                    is_password=self.hide_user_input_in_prompt,
+                    confirm=self.hide_user_input_in_prompt
                 )
             except NotImplementedError:
                 question.value = None
 
+        if getattr(self, "readonly", False):
+            msignals.display(self._format_text_for_user_input_in_cli(question))
+
         # we don't have an answer, check optional and default_value
         if question.value is None or question.value == "":
-            if not question.optional and question.default is None:
+            if not question.optional and question.default is None and check_required:
                 raise YunohostValidationError(
                     "app_argument_required", name=question.name
                 )
@@ -2785,6 +2822,7 @@ class YunoHostArgumentFormatParser(object):
         raise YunohostValidationError(
             "app_argument_choice_invalid",
             name=question.name,
+            value=question.value,
             choices=", ".join(question.choices),
         )
 
@@ -2796,7 +2834,15 @@ class YunoHostArgumentFormatParser(object):
 
         if question.default is not None:
             text_for_user_input_in_cli += " (default: {0})".format(question.default)
-
+        if question.help or question.helpLink:
+            text_for_user_input_in_cli += ":\033[m"
+        if question.help:
+            text_for_user_input_in_cli += "\n - "
+            text_for_user_input_in_cli += question.help['en']
+        if question.helpLink:
+            if not isinstance(question.helpLink, dict):
+                question.helpLink = {'href': question.helpLink}
+            text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}"
         return text_for_user_input_in_cli
 
     def _post_parse_value(self, question):
@@ -2884,6 +2930,7 @@ class BooleanArgumentParser(YunoHostArgumentFormatParser):
         raise YunohostValidationError(
             "app_argument_choice_invalid",
             name=question.name,
+            value=question.value,
             choices="yes, no, y, n, 1, 0",
         )
 
@@ -2967,13 +3014,73 @@ class NumberArgumentParser(YunoHostArgumentFormatParser):
 
 class DisplayTextArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "display_text"
+    readonly = True
 
-    def parse(self, question, user_answers):
-        print(question["ask"])
+    def parse_question(self, question, user_answers):
+        question = super(DisplayTextArgumentParser, self).parse_question(
+            question, user_answers
+        )
+
+        question.optional = True
+
+        return question
+
+    def _format_text_for_user_input_in_cli(self, question):
+        text = question.ask['en']
+        if question.type in ['info', 'warning', 'danger']:
+            color = {
+                'info': 'cyan',
+                'warning': 'yellow',
+                'danger': 'red'
+            }
+            return colorize(m18n.g(question.type), color[question.type]) + f" {text}"
+        else:
+            return text
 
 class FileArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "file"
 
+    def parse_question(self, question, user_answers):
+        question = super(FileArgumentParser, self).parse_question(
+            question, user_answers
+        )
+        if msettings.get('interface') == 'api':
+            question.value = {
+                'content': user_answers[question.name],
+                'filename': user_answers.get(f"{question.name}[name]", question.name),
+             } if user_answers[question.name] else None
+        return question
+
+    def _post_parse_value(self, question):
+        from base64 import b64decode
+        # Upload files from API
+        # A file arg contains a string with "FILENAME:BASE64_CONTENT"
+        if not question.value:
+            return question.value
+
+        if msettings.get('interface') == 'api':
+            upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
+            filename = question.value['filename']
+            logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}")
+
+            # Filename is given by user of the API. For security reason, we have replaced
+            # os.path.join to avoid the user to be able to rewrite a file in filesystem
+            # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
+            file_path = os.path.normpath(upload_dir + "/" + filename)
+            i = 2
+            while os.path.exists(file_path):
+                file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
+                i += 1
+            content = question.value['content']
+            try:
+                with open(file_path, 'wb') as f:
+                    f.write(b64decode(content))
+            except IOError as e:
+                raise YunohostError("cannot_write_file", file=file_path, error=str(e))
+            except Exception as e:
+                raise YunohostError("error_writing_file", file=file_path, error=str(e))
+            question.value = file_path
+        return question.value
 
 
 ARGUMENTS_TYPE_PARSERS = {
@@ -2994,11 +3101,16 @@ ARGUMENTS_TYPE_PARSERS = {
     "number": NumberArgumentParser,
     "range": NumberArgumentParser,
     "display_text": DisplayTextArgumentParser,
+    "success": DisplayTextArgumentParser,
+    "danger": DisplayTextArgumentParser,
+    "warning": DisplayTextArgumentParser,
+    "info": DisplayTextArgumentParser,
+    "markdown": DisplayTextArgumentParser,
     "file": FileArgumentParser,
 }
 
 
-def _parse_args_in_yunohost_format(user_answers, argument_questions):
+def _parse_args_in_yunohost_format(user_answers, argument_questions, check_required=True):
     """Parse arguments store in either manifest.json or actions.json or from a
     config panel against the user answers when they are present.
 
@@ -3014,7 +3126,7 @@ def _parse_args_in_yunohost_format(user_answers, argument_questions):
     for question in argument_questions:
         parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]()
 
-        answer = parser.parse(question=question, user_answers=user_answers)
+        answer = parser.parse(question=question, user_answers=user_answers, check_required=check_required)
         if answer is not None:
             parsed_answers_dict[question["name"]] = answer
 

From 2ac4e1c5bf37c0a33704da92c305a51fa5b94750 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 20 Aug 2021 17:26:26 +0200
Subject: [PATCH 20/51] [fix] I like regexp

---
 data/helpers.d/configpanel | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 5b290629d..efbd5248f 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -42,9 +42,9 @@ ynh_value_get() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
     
-    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)"
     
     local first_char="${crazy_value:0:1}"
     if [[ "$first_char" == '"' ]] ; then

From b79d5ae416ffaaf9fc8ddebf1db70dce0d462b21 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 20 Aug 2021 18:02:54 +0200
Subject: [PATCH 21/51] [enh] Support __FINALPATH__ in file source

---
 data/helpers.d/configpanel | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index efbd5248f..fcb1bd0d1 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -132,7 +132,7 @@ EOL
         elif [[ "$source" == *":"* ]] ; then
             local source_key="$(echo "$source" | cut -d: -f1)"
             source_key=${source_key:-$short_setting}
-            local source_file="$(echo "$source" | cut -d: -f2)"
+            local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
             old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")"
         
         # Specific case for files (all content of the file is the source)

From 10c8babf8c0ea2bfaab4e7aeb913a4750ed53c34 Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 20 Aug 2021 18:03:32 +0200
Subject: [PATCH 22/51] [enh] Fail if script fail

---
 src/yunohost/app.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index c489cceaa..676e07f5a 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1939,15 +1939,12 @@ ynh_panel_run $1
         "app_instance_nb": str(app_instance_nb),
     })
 
-    try:
-        _, parsed_values = hook_exec(
-            config_script, args=[action], env=env
-        )
-    except (KeyboardInterrupt, EOFError, Exception):
-        logger.error('Unable to extract some values')
-        parsed_values = {}
+    _, parsed_values = hook_exec(
+        config_script, args=[action], env=env
+    )
     return parsed_values
 
+
 def _get_all_installed_apps_id():
     """
     Return something like:

From ef058c07f7fd011ff605b13780e933dc3772a8bc Mon Sep 17 00:00:00 2001
From: ljf <ljf+github@grimaud.me>
Date: Fri, 20 Aug 2021 20:12:15 +0200
Subject: [PATCH 23/51] [fix] File question in config panel with cli

---
 data/helpers.d/configpanel | 41 ++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index fcb1bd0d1..67c7a92f7 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -44,7 +44,8 @@ ynh_value_get() {
 
     local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
     
-    local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)"
+    local crazy_value="$((grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL)  | head -n1)"
+    #"
     
     local first_char="${crazy_value:0:1}"
     if [[ "$first_char" == '"' ]] ; then
@@ -74,22 +75,22 @@ ynh_value_set() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    local var_part="[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
     
-    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    local crazy_value="$(grep -i -o -P "${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
     local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
     local first_char="${crazy_value:0:1}"
     if [[ "$first_char" == '"' ]] ; then
         value="$(echo "$value" | sed 's/"/\\"/g')"
-        sed -ri "s%^(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+        sed -ri "s%(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file}
     elif [[ "$first_char" == "'" ]] ; then
         value="$(echo "$value" | sed "s/'/\\\\'/g")"
-        sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+        sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
     else
         if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
             value="\"$(echo "$value" | sed 's/"/\\"/g')\""
         fi
-        sed -ri "s%^(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+        sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
     fi
 }
 
@@ -124,12 +125,12 @@ EOL
         if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
             old[$short_setting]="$($getter)"
         
-        # Get value from app settings
-        elif [[ "$source" == "settings" ]] ; then
-            old[$short_setting]="$(ynh_app_setting_get $app $short_setting)"
+        # Get value from app settings or from another file
+        elif [[ "$source" == "settings" ]] || [[ "$source" == *":"* ]] ; then
+            if [[ "$source" == "settings" ]] ; then
+                source=":/etc/yunohost/apps/$app/settings.yml"
+            fi
         
-        # Get value from a kind of key/value file
-        elif [[ "$source" == *":"* ]] ; then
             local source_key="$(echo "$source" | cut -d: -f1)"
             source_key=${source_key:-$short_setting}
             local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
@@ -137,7 +138,8 @@ EOL
         
         # Specific case for files (all content of the file is the source)
         else
-            old[$short_setting]="$source"
+            
+            old[$short_setting]="$(ls $source 2> /dev/null || echo YNH_NULL)"
             file_hash[$short_setting]="true"
         fi
     done
@@ -164,12 +166,13 @@ _ynh_panel_apply() {
             then
                 local source_key="$(echo "$source" | cut -d: -f1)"
                 source_key=${source_key:-$short_setting}
-                local source_file="$(echo "$source" | cut -d: -f2)"
+                local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
                 ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}"
 
             # Specific case for files (all content of the file is the source)
             else
-                cp "${!short_setting}" "$source"
+                local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+                cp "${!short_setting}" "$source_file"
             fi
         fi
     done
@@ -178,7 +181,9 @@ _ynh_panel_apply() {
 _ynh_panel_show() {
     for short_setting in "${!old[@]}"
     do
-	    ynh_return "${short_setting}: \"${old[$short_setting]}\""
+        if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then
+	        ynh_return "${short_setting}: \"${old[$short_setting]}\""
+        fi
     done
 }
 
@@ -197,10 +202,11 @@ _ynh_panel_validate() {
                 file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
             fi
             if [ -f "${!short_setting}" ] ; then
-                file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1)
+                file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1)
                 if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
                 then
                     changed[$short_setting]=true
+                    is_error=false
                 fi
             fi
         else
@@ -218,6 +224,7 @@ _ynh_panel_validate() {
 
         for short_setting in "${!old[@]}"
         do
+            [[ "${changed[$short_setting]}" == "false" ]] && continue
             local result=""
             if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then
                 result="$(validate__$short_setting)"
@@ -225,7 +232,7 @@ _ynh_panel_validate() {
             if [ -n "$result" ]
             then
                 local key="YNH_ERROR_${$short_setting}"
-                ynh_return "$key=$result"
+                ynh_return "$key: $result"
                 is_error=true
             fi
         done

From 0de69104b153bf24b18f533c89f1f7b007734a18 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 24 Aug 2021 16:06:46 +0200
Subject: [PATCH 24/51] [fix] Bad merge

---
 data/helpers.d/configpanel | 51 --------------------------------------
 1 file changed, 51 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index e8dbaafe9..67c7a92f7 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -181,39 +181,26 @@ _ynh_panel_apply() {
 _ynh_panel_show() {
     for short_setting in "${!old[@]}"
     do
-<<<<<<< HEAD
         if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then
 	        ynh_return "${short_setting}: \"${old[$short_setting]}\""
         fi
-=======
-        local key="YNH_CONFIG_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
-	    ynh_return "$key=${old[$short_setting]}"
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
     done
 }
 
 _ynh_panel_validate() {
-<<<<<<< HEAD
-=======
-    set +x
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
     # Change detection
     local is_error=true
     #for changed_status in "${!changed[@]}"
     for short_setting in "${!old[@]}"
     do
         changed[$short_setting]=false
-<<<<<<< HEAD
         [ -z ${!short_setting+x} ] && continue
-=======
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
         if [ ! -z "${file_hash[${short_setting}]}" ] ; then
             file_hash[old__$short_setting]=""
             file_hash[new__$short_setting]=""
             if [ -f "${old[$short_setting]}" ] ; then
                 file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
             fi
-<<<<<<< HEAD
             if [ -f "${!short_setting}" ] ; then
                 file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1)
                 if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
@@ -224,17 +211,6 @@ _ynh_panel_validate() {
             fi
         else
             if [[ "${!short_setting}" != "${old[$short_setting]}" ]]
-=======
-            if [ -f "${new[$short_setting]}" ] ; then
-                file_hash[new__$short_setting]=$(sha256sum "${new[$short_setting]}" | cut -d' ' -f1)
-                if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
-                then
-                    changed[$short_setting]=true
-                fi
-            fi
-        else
-            if [[ "${new[$short_setting]}" != "${old[$short_setting]}" ]]
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
             then
                 changed[$short_setting]=true
                 is_error=false
@@ -248,23 +224,15 @@ _ynh_panel_validate() {
 
         for short_setting in "${!old[@]}"
         do
-<<<<<<< HEAD
             [[ "${changed[$short_setting]}" == "false" ]] && continue
-=======
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
             local result=""
             if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then
                 result="$(validate__$short_setting)"
             fi
             if [ -n "$result" ]
             then
-<<<<<<< HEAD
                 local key="YNH_ERROR_${$short_setting}"
                 ynh_return "$key: $result"
-=======
-                local key="YNH_ERROR_$(ynh_lowerdot_to_uppersnake $dot_settings[$short_setting])"
-                ynh_return "$key=$result"
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
                 is_error=true
             fi
         done
@@ -272,14 +240,8 @@ _ynh_panel_validate() {
 
     if [[ "$is_error" == "true" ]]
     then
-<<<<<<< HEAD
         ynh_die ""
     fi 
-=======
-        ynh_die
-    fi 
-    set -x
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
     
 }
 
@@ -301,7 +263,6 @@ ynh_panel_apply() {
 
 ynh_panel_run() {
     declare -Ag old=()
-<<<<<<< HEAD
     declare -Ag changed=()
     declare -Ag file_hash=()
     declare -Ag sources=()
@@ -310,18 +271,6 @@ ynh_panel_run() {
     case $1 in
         show) ynh_panel_get && ynh_panel_show;;
         apply) ynh_panel_get && ynh_panel_validate && ynh_panel_apply;;
-=======
-    declare -Ag new=()
-    declare -Ag changed=()
-    declare -Ag file_hash=()
-    declare -Ag sources=()
-    declare -Ag dot_settings=()
-    
-    ynh_panel_get
-    case $1 in
-        show) ynh_panel_show;;
-        apply) ynh_panel_validate && ynh_panel_apply;;
->>>>>>> 596d05ae81d24712c87ec0de72f8deb8248bca9a
     esac
 }
 

From 4c46f41036b505d51faeab7f1545e7f3db421e1e Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 24 Aug 2021 17:36:54 +0200
Subject: [PATCH 25/51] [fix] Args always empty in config panel

---
 src/yunohost/app.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 676e07f5a..770706190 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1861,11 +1861,13 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
     operation_logger.start()
 
     # Prepare pre answered questions
-    args = {}
     if args:
         args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
     elif value is not None:
         args = {key: value}
+    else:
+        args = {}
+
 
     upload_dir = None
 

From 4547a6ec077f7d09a7334a08b92ccc7da4ba2827 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 25 Aug 2021 18:13:17 +0200
Subject: [PATCH 26/51] [fix] Multiple fixes in config panel

---
 data/helpers.d/configpanel | 73 ++++++++++++++++++++++++--------------
 src/yunohost/app.py        | 16 +++++----
 2 files changed, 55 insertions(+), 34 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 67c7a92f7..7e26811f5 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -189,12 +189,18 @@ _ynh_panel_show() {
 
 _ynh_panel_validate() {
     # Change detection
+    ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1
     local is_error=true
     #for changed_status in "${!changed[@]}"
     for short_setting in "${!old[@]}"
     do
         changed[$short_setting]=false
-        [ -z ${!short_setting+x} ] && continue
+        if [ -z ${!short_setting+x} ]; then
+            # Assign the var with the old value in order to allows multiple
+            # args validation
+            declare "$short_setting"="${old[$short_setting]}"
+            continue
+        fi
         if [ ! -z "${file_hash[${short_setting}]}" ] ; then
             file_hash[old__$short_setting]=""
             file_hash[new__$short_setting]=""
@@ -217,30 +223,32 @@ _ynh_panel_validate() {
             fi
         fi
     done
-    
-    # Run validation if something is changed
-    if [[ "$is_error" == "false" ]]
-    then
-
-        for short_setting in "${!old[@]}"
-        do
-            [[ "${changed[$short_setting]}" == "false" ]] && continue
-            local result=""
-            if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then
-                result="$(validate__$short_setting)"
-            fi
-            if [ -n "$result" ]
-            then
-                local key="YNH_ERROR_${$short_setting}"
-                ynh_return "$key: $result"
-                is_error=true
-            fi
-        done
-    fi
-
     if [[ "$is_error" == "true" ]]
     then
-        ynh_die ""
+        ynh_die "Nothing has changed"
+    fi 
+    
+    # Run validation if something is changed
+    ynh_script_progression --message="Validating the new configuration..." --weight=1
+
+    for short_setting in "${!old[@]}"
+    do
+        [[ "${changed[$short_setting]}" == "false" ]] && continue
+        local result=""
+        if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then
+            result="$(validate__$short_setting)"
+        fi
+        if [ -n "$result" ]
+        then
+            local key="YNH_ERROR_${short_setting}"
+            ynh_return "$key: $result"
+            is_error=true
+        fi
+    done
+    
+    if [[ "$is_error" == "true" ]]
+    then
+        exit 0
     fi 
     
 }
@@ -266,11 +274,22 @@ ynh_panel_run() {
     declare -Ag changed=()
     declare -Ag file_hash=()
     declare -Ag sources=()
-    
-    ynh_panel_get
     case $1 in
-        show) ynh_panel_get && ynh_panel_show;;
-        apply) ynh_panel_get && ynh_panel_validate && ynh_panel_apply;;
+        show)
+            ynh_panel_get
+            ynh_panel_show
+            ;;
+        apply) 
+            max_progression=4
+            ynh_script_progression --message="Reading config panel description and current configuration..." --weight=1
+            ynh_panel_get
+            
+            ynh_panel_validate
+            
+            ynh_script_progression --message="Applying the new configuration..." --weight=1
+            ynh_panel_apply
+            ynh_script_progression --message="Configuration of $app completed" --last
+            ;;
     esac
 }
 
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 770706190..92254d1d0 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1769,7 +1769,7 @@ def app_config_show(operation_logger, app, panel='', full=False):
         return None
 
     # Call config script to extract current values
-    parsed_values = _call_config_script(app, 'show')
+    parsed_values = _call_config_script(operation_logger, app, 'show')
 
     # # Check and transform values if needed
     # options = [option for _, _, option in _get_options_iterator(config_panel)]
@@ -1820,7 +1820,7 @@ def app_config_get(operation_logger, app, key):
         raise YunohostError("app_config_no_panel")
 
     # Call config script to extract current values
-    parsed_values = _call_config_script(app, 'show')
+    parsed_values = _call_config_script(operation_logger, app, 'show')
 
     logger.debug("Searching value")
     short_key = key.split('.')[-1]
@@ -1891,7 +1891,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
     env = {key: value[0] for key, value in args_dict.items()}
 
     try:
-        errors = _call_config_script(app, 'apply', env=env)
+        errors = _call_config_script(operation_logger, app, 'apply', env=env)
     # Here again, calling hook_exec could fail miserably, or get
     # manually interrupted (by mistake or because script was stuck)
     except (KeyboardInterrupt, EOFError, Exception):
@@ -1917,7 +1917,7 @@ def _get_options_iterator(config_panel):
                 yield (panel, section, option)
 
 
-def _call_config_script(app, action, env={}):
+def _call_config_script(operation_logger, app, action, env={}):
     from yunohost.hook import hook_exec
 
     # Add default config script if needed
@@ -1933,7 +1933,7 @@ ynh_panel_run $1
         write_to_file(config_script, default_script)
 
     # Call config script to extract current values
-    logger.debug("Calling 'show' action from config script")
+    logger.debug(f"Calling '{action}' action from config script")
     app_id, app_instance_nb = _parse_app_instance_name(app)
     env.update({
         "app_id": app_id,
@@ -1941,9 +1941,11 @@ ynh_panel_run $1
         "app_instance_nb": str(app_instance_nb),
     })
 
-    _, parsed_values = hook_exec(
+    ret, parsed_values = hook_exec(
         config_script, args=[action], env=env
     )
+    if ret != 0:
+        operation_logger.error(parsed_values)
     return parsed_values
 
 
@@ -3047,7 +3049,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
             question.value = {
                 'content': user_answers[question.name],
                 'filename': user_answers.get(f"{question.name}[name]", question.name),
-             } if user_answers[question.name] else None
+             } if user_answers.get(question.name) else None
         return question
 
     def _post_parse_value(self, question):

From 86c099812363d96142d6fdb9a53b2168365a247a Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 25 Aug 2021 19:18:15 +0200
Subject: [PATCH 27/51] [enh] Add a pattern validation for config panel

---
 src/yunohost/app.py | 74 ++++++++++++++++++++++++++++-----------------
 1 file changed, 46 insertions(+), 28 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 92254d1d0..1853a63f4 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -2764,6 +2764,7 @@ class YunoHostArgumentFormatParser(object):
         parsed_question.optional = question.get("optional", False)
         parsed_question.ask = question.get("ask")
         parsed_question.help = question.get("help")
+        parsed_question.pattern = question.get("pattern")
         parsed_question.helpLink = question.get("helpLink")
         parsed_question.value = user_answers.get(parsed_question.name)
 
@@ -2776,49 +2777,66 @@ class YunoHostArgumentFormatParser(object):
 
         return parsed_question
 
-    def parse(self, question, user_answers, check_required=True):
+    def parse(self, question, user_answers):
         question = self.parse_question(question, user_answers)
 
-        if question.value is None and not getattr(self, "readonly", False):
-            text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
-                question
-            )
-            try:
-                question.value = msignals.prompt(
-                    message=text_for_user_input_in_cli,
-                    is_password=self.hide_user_input_in_prompt,
-                    confirm=self.hide_user_input_in_prompt
+        while True:
+            # Display question if no value filled or if it's a readonly message
+            if msettings.get('interface') == 'cli':
+                text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
+                    question
                 )
-            except NotImplementedError:
-                question.value = None
+                if getattr(self, "readonly", False):
+                    msignals.display(text_for_user_input_in_cli)
 
-        if getattr(self, "readonly", False):
-            msignals.display(self._format_text_for_user_input_in_cli(question))
+                elif question.value is None:
+                    question.value = msignals.prompt(
+                        message=text_for_user_input_in_cli,
+                        is_password=self.hide_user_input_in_prompt,
+                        confirm=self.hide_user_input_in_prompt
+                    )
 
-        # we don't have an answer, check optional and default_value
-        if question.value is None or question.value == "":
-            if not question.optional and question.default is None and check_required:
-                raise YunohostValidationError(
-                    "app_argument_required", name=question.name
-                )
-            else:
+
+            # Apply default value
+            if question.value in [None, ""] and question.default is not None:
                 question.value = (
                     getattr(self, "default_value", None)
                     if question.default is None
                     else question.default
                 )
 
-        # we have an answer, do some post checks
-        if question.value is not None:
-            if question.choices and question.value not in question.choices:
-                self._raise_invalid_answer(question)
-
+            # Prevalidation
+            try:
+                self._prevalidate(question)
+            except YunoHostValidationError:
+                if msettings.get('interface') == 'api':
+                    raise
+                question.value = None
+                continue
+            break
         # this is done to enforce a certain formating like for boolean
         # by default it doesn't do anything
         question.value = self._post_parse_value(question)
 
         return (question.value, self.argument_type)
 
+    def _prevalidate(self, question):
+        if question.value in [None, ""] and not question.optional:
+            raise YunohostValidationError(
+                "app_argument_required", name=question.name
+            )
+
+        # we have an answer, do some post checks
+        if question.value is not None:
+            if question.choices and question.value not in question.choices:
+                self._raise_invalid_answer(question)
+            if question.pattern and re.match(question.pattern['regexp'], str(question.value)):
+                raise YunohostValidationError(
+                    question.pattern['error'],
+                    name=question.name,
+                    value=question.value,
+                )
+
     def _raise_invalid_answer(self, question):
         raise YunohostValidationError(
             "app_argument_choice_invalid",
@@ -3111,7 +3129,7 @@ ARGUMENTS_TYPE_PARSERS = {
 }
 
 
-def _parse_args_in_yunohost_format(user_answers, argument_questions, check_required=True):
+def _parse_args_in_yunohost_format(user_answers, argument_questions):
     """Parse arguments store in either manifest.json or actions.json or from a
     config panel against the user answers when they are present.
 
@@ -3127,7 +3145,7 @@ def _parse_args_in_yunohost_format(user_answers, argument_questions, check_requi
     for question in argument_questions:
         parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]()
 
-        answer = parser.parse(question=question, user_answers=user_answers, check_required=check_required)
+        answer = parser.parse(question=question, user_answers=user_answers)
         if answer is not None:
             parsed_answers_dict[question["name"]] = answer
 

From 5dc1ee62660c2b06be840ddd20cf6b051ace86f0 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 25 Aug 2021 20:10:11 +0200
Subject: [PATCH 28/51] [enh] Services key in config panel

---
 src/yunohost/app.py | 39 ++++++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 1853a63f4..3d775ea5b 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -54,7 +54,7 @@ from moulinette.utils.filesystem import (
     mkdir,
 )
 
-from yunohost.service import service_status, _run_service_command
+from yunohost.service import service_status, _run_service_command, _get_services
 from yunohost.utils import packages
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.filesystem import free_space_in_directory
@@ -1870,9 +1870,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
 
 
     upload_dir = None
-
     for panel in config_panel.get("panel", []):
-
         if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
             msignals.display(colorize("\n" + "=" * 40, 'purple'))
             msignals.display(colorize(f">>>> {panel['name']}", 'purple'))
@@ -1902,7 +1900,29 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
             if upload_dir is not None:
                 shutil.rmtree(upload_dir)
 
-    logger.success("Config updated as expected")
+    # Reload services
+    services_to_reload = set([])
+    for panel in config_panel.get("panel", []):
+        services_to_reload |= set(panel.get('services', []))
+        for section in panel.get("sections", []):
+            services_to_reload |= set(section.get('services', []))
+            for option in section.get("options", []):
+                services_to_reload |= set(section.get('options', []))
+
+    services_to_reload = list(services_to_reload)
+    services_to_reload.sort(key = 'nginx'.__eq__)
+    for service in services_to_reload:
+        if not _run_service_command('reload_or_restart', service):
+            services = _get_services()
+            test_conf = services[service].get('test_conf')
+            errors = check_output(f"{test_conf}; exit 0") if test_conf else ''
+            raise YunohostError(
+                "app_config_failed_service_reload",
+                service=service, errors=errors
+            )
+
+    if not errors:
+        logger.success("Config updated as expected")
     return {
         "app": app,
         "errors": errors,
@@ -2760,17 +2780,14 @@ class YunoHostArgumentFormatParser(object):
         parsed_question.name = question["name"]
         parsed_question.type = question.get("type", 'string')
         parsed_question.default = question.get("default", None)
-        parsed_question.choices = question.get("choices", [])
         parsed_question.optional = question.get("optional", False)
-        parsed_question.ask = question.get("ask")
-        parsed_question.help = question.get("help")
+        parsed_question.choices = question.get("choices", [])
         parsed_question.pattern = question.get("pattern")
+        parsed_question.ask = question.get("ask", {'en': f"Enter value for '{parsed_question.name}':"})
+        parsed_question.help = question.get("help")
         parsed_question.helpLink = question.get("helpLink")
         parsed_question.value = user_answers.get(parsed_question.name)
 
-        if parsed_question.ask is None:
-            parsed_question.ask = "Enter value for '%s':" % parsed_question.name
-
         # Empty value is parsed as empty string
         if parsed_question.default == "":
             parsed_question.default = None
@@ -2857,7 +2874,7 @@ class YunoHostArgumentFormatParser(object):
             text_for_user_input_in_cli += ":\033[m"
         if question.help:
             text_for_user_input_in_cli += "\n - "
-            text_for_user_input_in_cli += question.help['en']
+            text_for_user_input_in_cli += _value_for_locale(question.help)
         if question.helpLink:
             if not isinstance(question.helpLink, dict):
                 question.helpLink = {'href': question.helpLink}

From 97128d7ddbaaf4806c6bfe1d5e9847c78aa9c0c0 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 25 Aug 2021 20:16:31 +0200
Subject: [PATCH 29/51] [enh] Support __APP__ in services config panel key

---
 src/yunohost/app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 3d775ea5b..fba8499c0 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1912,6 +1912,8 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
     services_to_reload = list(services_to_reload)
     services_to_reload.sort(key = 'nginx'.__eq__)
     for service in services_to_reload:
+        if service == "__APP__":
+            service = app
         if not _run_service_command('reload_or_restart', service):
             services = _get_services()
             test_conf = services[service].get('test_conf')

From 5a64a063b285d99e84bd22097cbae8f63a74121a Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 25 Aug 2021 20:51:50 +0200
Subject: [PATCH 30/51] [fix] Clean properly config panel upload dir

---
 src/yunohost/app.py | 71 +++++++++++++++++++++++++--------------------
 1 file changed, 40 insertions(+), 31 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index fba8499c0..aa8f0d864 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1861,34 +1861,29 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
     operation_logger.start()
 
     # Prepare pre answered questions
-    if args:
-        args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
-    elif value is not None:
+    args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
+    if value is not None:
         args = {key: value}
-    else:
-        args = {}
-
-
-    upload_dir = None
-    for panel in config_panel.get("panel", []):
-        if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
-            msignals.display(colorize("\n" + "=" * 40, 'purple'))
-            msignals.display(colorize(f">>>> {panel['name']}", 'purple'))
-            msignals.display(colorize("=" * 40, 'purple'))
-        for section in panel.get("sections", []):
-            if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
-                msignals.display(colorize(f"\n# {section['name']}", 'purple'))
-
-            # Check and ask unanswered questions
-            args_dict = _parse_args_in_yunohost_format(
-                args, section['options']
-            )
-
-    # Call config script to extract current values
-    logger.info("Running config script...")
-    env = {key: value[0] for key, value in args_dict.items()}
 
     try:
+        for panel in config_panel.get("panel", []):
+            if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
+                msignals.display(colorize("\n" + "=" * 40, 'purple'))
+                msignals.display(colorize(f">>>> {panel['name']}", 'purple'))
+                msignals.display(colorize("=" * 40, 'purple'))
+            for section in panel.get("sections", []):
+                if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
+                    msignals.display(colorize(f"\n# {section['name']}", 'purple'))
+
+                # Check and ask unanswered questions
+                args_dict = _parse_args_in_yunohost_format(
+                    args, section['options']
+                )
+
+        # Call config script to extract current values
+        logger.info("Running config script...")
+        env = {key: value[0] for key, value in args_dict.items()}
+
         errors = _call_config_script(operation_logger, app, 'apply', env=env)
     # Here again, calling hook_exec could fail miserably, or get
     # manually interrupted (by mistake or because script was stuck)
@@ -1896,11 +1891,16 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
         raise YunohostError("unexpected_error")
     finally:
         # Delete files uploaded from API
-        if msettings.get('interface') == 'api':
-            if upload_dir is not None:
-                shutil.rmtree(upload_dir)
+        FileArgumentParser.clean_upload_dirs()
+
+    if errors:
+        return {
+            "app": app,
+            "errors": errors,
+        }
 
     # Reload services
+    logger.info("Reloading services...")
     services_to_reload = set([])
     for panel in config_panel.get("panel", []):
         services_to_reload |= set(panel.get('services', []))
@@ -1923,11 +1923,10 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
                 service=service, errors=errors
             )
 
-    if not errors:
-        logger.success("Config updated as expected")
+    logger.success("Config updated as expected")
     return {
         "app": app,
-        "errors": errors,
+        "errors": [],
         "logs": operation_logger.success(),
     }
 
@@ -3077,6 +3076,14 @@ class DisplayTextArgumentParser(YunoHostArgumentFormatParser):
 
 class FileArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "file"
+    upload_dirs = []
+
+    @classmethod
+    def clean_upload_dirs(cls):
+        # Delete files uploaded from API
+        if msettings.get('interface') == 'api':
+            for upload_dir in cls.upload_dirs:
+                shutil.rmtree(upload_dir)
 
     def parse_question(self, question, user_answers):
         question = super(FileArgumentParser, self).parse_question(
@@ -3097,7 +3104,9 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
             return question.value
 
         if msettings.get('interface') == 'api':
+
             upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
+            FileArgumentParser.upload_dirs += [upload_dir]
             filename = question.value['filename']
             logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}")
 

From 8d364029a03bbaf87f9cc491e3e7d5975a1f49bc Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 25 Aug 2021 20:58:15 +0200
Subject: [PATCH 31/51] [fix] Services key not properly converted

---
 src/yunohost/app.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index aa8f0d864..f94e22462 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1866,6 +1866,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
         args = {key: value}
 
     try:
+        logger.debug("Asking unanswered question and prevalidating...")
         for panel in config_panel.get("panel", []):
             if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
                 msignals.display(colorize("\n" + "=" * 40, 'purple'))
@@ -2172,6 +2173,7 @@ def _get_app_config_panel(app_id, filter_key=''):
             panel = {
                 "id": key,
                 "name": value.get("name", ""),
+                "services": value.get("services", []),
                 "sections": [],
             }
 
@@ -2190,6 +2192,7 @@ def _get_app_config_panel(app_id, filter_key=''):
                     "id": section_key,
                     "name": section_value.get("name", ""),
                     "optional": section_value.get("optional", True),
+                    "services": value.get("services", []),
                     "options": [],
                 }
 

From f5529c584d8fd4d91a9177f2fa9f946790011b0c Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Wed, 25 Aug 2021 21:16:31 +0200
Subject: [PATCH 32/51] [wip] Min max in number questions

---
 src/yunohost/app.py | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index f94e22462..44c93e6d9 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1908,14 +1908,15 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
         for section in panel.get("sections", []):
             services_to_reload |= set(section.get('services', []))
             for option in section.get("options", []):
-                services_to_reload |= set(section.get('options', []))
+                services_to_reload |= set(option.get('services', []))
 
     services_to_reload = list(services_to_reload)
     services_to_reload.sort(key = 'nginx'.__eq__)
     for service in services_to_reload:
         if service == "__APP__":
             service = app
-        if not _run_service_command('reload_or_restart', service):
+        logger.debug(f"Reloading {service}")
+        if not _run_service_command('reload-or-restart', service):
             services = _get_services()
             test_conf = services[service].get('test_conf')
             errors = check_output(f"{test_conf}; exit 0") if test_conf else ''
@@ -2937,7 +2938,7 @@ class BooleanArgumentParser(YunoHostArgumentFormatParser):
     default_value = False
 
     def parse_question(self, question, user_answers):
-        question = super(BooleanArgumentParser, self).parse_question(
+        question = super().parse_question(
             question, user_answers
         )
 
@@ -3031,18 +3032,33 @@ class NumberArgumentParser(YunoHostArgumentFormatParser):
     default_value = ""
 
     def parse_question(self, question, user_answers):
-        question = super(NumberArgumentParser, self).parse_question(
+        question_parsed = super().parse_question(
             question, user_answers
         )
-
+        question_parsed.min = question.get('min', None)
+        question_parsed.max = question.get('max', None)
         if question.default is None:
-            question.default = 0
+            question_parsed.default = 0
 
-        return question
+        return question_parsed
 
+    def _prevalidate(self, question):
+        super()._prevalidate(question)
+        if question.min is not None and question.value < question.min:
+            raise YunohostValidationError(
+                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+            )
+        if question.max is not None and question.value > question.max:
+            raise YunohostValidationError(
+                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+            )
+        if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()):
+            raise YunohostValidationError(
+                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+            )
     def _post_parse_value(self, question):
         if isinstance(question.value, int):
-            return super(NumberArgumentParser, self)._post_parse_value(question)
+            return super()._post_parse_value(question)
 
         if isinstance(question.value, str) and question.value.isdigit():
             return int(question.value)

From 9eb9ec1804e45856b07962e16393f0100bf6d113 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Thu, 26 Aug 2021 17:39:33 +0200
Subject: [PATCH 33/51] [fix] Several fixes in config panel

---
 src/yunohost/app.py | 59 +++++++++++++++++++++++++++++++++------------
 1 file changed, 43 insertions(+), 16 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 44c93e6d9..0e945c64c 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -70,6 +70,7 @@ APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml"
 APPS_CATALOG_API_VERSION = 2
 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
 
+APPS_CONFIG_PANEL_VERSION_SUPPORTED = 1.0
 re_app_instance_name = re.compile(
     r"^(?P<appid>[\w-]+?)(__(?P<appinstancenb>[1-9][0-9]*))?$"
 )
@@ -1863,7 +1864,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
     # Prepare pre answered questions
     args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
     if value is not None:
-        args = {key: value}
+        args = {filter_key.split('.')[-1]: value}
 
     try:
         logger.debug("Asking unanswered question and prevalidating...")
@@ -1883,13 +1884,23 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
 
         # Call config script to extract current values
         logger.info("Running config script...")
-        env = {key: value[0] for key, value in args_dict.items()}
+        env = {key: str(value[0]) for key, value in args_dict.items()}
 
         errors = _call_config_script(operation_logger, app, 'apply', env=env)
-    # Here again, calling hook_exec could fail miserably, or get
-    # manually interrupted (by mistake or because script was stuck)
-    except (KeyboardInterrupt, EOFError, Exception):
-        raise YunohostError("unexpected_error")
+    # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception
+    except (KeyboardInterrupt, EOFError):
+        error = m18n.n("operation_interrupted")
+        logger.error(m18n.n("app_config_failed", app=app, error=error))
+        failure_message_with_debug_instructions = operation_logger.error(error)
+        raise
+    # Something wrong happened in Yunohost's code (most probably hook_exec)
+    except Exception:
+        import traceback
+
+        error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
+        logger.error(m18n.n("app_config_failed", app=app, error=error))
+        failure_message_with_debug_instructions = operation_logger.error(error)
+        raise
     finally:
         # Delete files uploaded from API
         FileArgumentParser.clean_upload_dirs()
@@ -2148,6 +2159,11 @@ def _get_app_config_panel(app_id, filter_key=''):
         toml_config_panel = toml.load(
             open(config_panel_toml_path, "r"), _dict=OrderedDict
         )
+        if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED:
+            raise YunohostError(
+                "app_config_too_old_version", app=app_id,
+                version=toml_config_panel["version"]
+            )
 
         # transform toml format into json format
         config_panel = {
@@ -2219,6 +2235,13 @@ def _get_app_config_panel(app_id, filter_key=''):
 
             config_panel["panel"].append(panel)
 
+        if (filter_panel and len(config_panel['panel']) == 0) or \
+           (filter_section and len(config_panel['panel'][0]['sections']) == 0) or \
+           (filter_option and len(config_panel['panel'][0]['sections'][0]['options']) == 0):
+            raise YunohostError(
+                "app_config_bad_filter_key", app=app_id, filter_key=filter_key
+            )
+
         return config_panel
 
     return None
@@ -2830,9 +2853,10 @@ class YunoHostArgumentFormatParser(object):
             # Prevalidation
             try:
                 self._prevalidate(question)
-            except YunoHostValidationError:
+            except YunohostValidationError as e:
                 if msettings.get('interface') == 'api':
                     raise
+                msignals.display(str(e), 'error')
                 question.value = None
                 continue
             break
@@ -3037,25 +3061,28 @@ class NumberArgumentParser(YunoHostArgumentFormatParser):
         )
         question_parsed.min = question.get('min', None)
         question_parsed.max = question.get('max', None)
-        if question.default is None:
+        if question_parsed.default is None:
             question_parsed.default = 0
 
         return question_parsed
 
     def _prevalidate(self, question):
         super()._prevalidate(question)
-        if question.min is not None and question.value < question.min:
-            raise YunohostValidationError(
-                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
-            )
-        if question.max is not None and question.value > question.max:
-            raise YunohostValidationError(
-                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
-            )
         if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()):
             raise YunohostValidationError(
                 "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
             )
+
+        if question.min is not None and int(question.value) < question.min:
+            raise YunohostValidationError(
+                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+            )
+
+        if question.max is not None and int(question.value) > question.max:
+            raise YunohostValidationError(
+                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+            )
+
     def _post_parse_value(self, question):
         if isinstance(question.value, int):
             return super()._post_parse_value(question)

From 574b01bcf4ae164518333220d4fd9fdc8964aa24 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Thu, 26 Aug 2021 17:48:08 +0200
Subject: [PATCH 34/51] [fix] Pattern key not working

---
 src/yunohost/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 0e945c64c..b5b4128dc 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -2876,7 +2876,7 @@ class YunoHostArgumentFormatParser(object):
         if question.value is not None:
             if question.choices and question.value not in question.choices:
                 self._raise_invalid_answer(question)
-            if question.pattern and re.match(question.pattern['regexp'], str(question.value)):
+            if question.pattern and not re.match(question.pattern['regexp'], str(question.value)):
                 raise YunohostValidationError(
                     question.pattern['error'],
                     name=question.name,

From b0f6a07372f4e5d5ebcaa37c4c9a5dad6c674713 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 27 Aug 2021 01:05:20 +0200
Subject: [PATCH 35/51] [fix] Source getter for config panel

---
 data/helpers.d/configpanel | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 7e26811f5..e1a96b866 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -139,7 +139,7 @@ EOL
         # Specific case for files (all content of the file is the source)
         else
             
-            old[$short_setting]="$(ls $source 2> /dev/null || echo YNH_NULL)"
+            old[$short_setting]="$(ls $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)"
             file_hash[$short_setting]="true"
         fi
     done

From bc725e9768639bcf4330627156af9b84c750f1f8 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 27 Aug 2021 01:05:59 +0200
Subject: [PATCH 36/51] [fix] File through config panel API

---
 src/yunohost/app.py | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index b5b4128dc..4e20c6950 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -2937,7 +2937,7 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser):
 
         return question
 
-    def _post_parse_value(self, question):
+    def _prevalidate(self, question):
         if any(char in question.value for char in self.forbidden_chars):
             raise YunohostValidationError(
                 "pattern_password_app", forbidden_chars=self.forbidden_chars
@@ -2949,8 +2949,6 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser):
 
             assert_password_is_strong_enough("user", question.value)
 
-        return super(PasswordArgumentParser, self)._post_parse_value(question)
-
 
 class PathArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "path"
@@ -3129,18 +3127,40 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
         # Delete files uploaded from API
         if msettings.get('interface') == 'api':
             for upload_dir in cls.upload_dirs:
-                shutil.rmtree(upload_dir)
+                if os.path.exists(upload_dir):
+                    shutil.rmtree(upload_dir)
 
     def parse_question(self, question, user_answers):
-        question = super(FileArgumentParser, self).parse_question(
+        question_parsed = super().parse_question(
             question, user_answers
         )
+        if question.get('accept'):
+            question_parsed.accept = question.get('accept').replace(' ', '').split(',')
+        else:
+            question.accept = []
         if msettings.get('interface') == 'api':
-            question.value = {
-                'content': user_answers[question.name],
-                'filename': user_answers.get(f"{question.name}[name]", question.name),
-             } if user_answers.get(question.name) else None
-        return question
+            if user_answers.get(question_parsed.name):
+                question_parsed.value = {
+                    'content': question_parsed.value,
+                    'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name),
+                }
+        return question_parsed
+
+    def _prevalidate(self, question):
+        super()._prevalidate(question)
+        if isinstance(question.value, str) and not os.path.exists(question.value):
+            raise YunohostValidationError(
+                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number1")
+            )
+        if question.value is None or not question.accept:
+            return
+
+        filename = question.value if isinstance(question.value, str) else question.value['filename']
+        if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept:
+            raise YunohostValidationError(
+                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number2")
+            )
+
 
     def _post_parse_value(self, question):
         from base64 import b64decode

From f1e5309d40a429513354e2c79857ee7d6623a8df Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 30 Aug 2021 13:14:17 +0200
Subject: [PATCH 37/51] Multiline, file, tags management + prefilled cli

---
 data/actionsmap/yunohost.yml |   4 +-
 data/helpers.d/configpanel   | 112 ++++++++++++++++++--------
 src/yunohost/app.py          | 147 +++++++++++++++++++++++------------
 3 files changed, 177 insertions(+), 86 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index d9e3a50d0..28b713b03 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -838,8 +838,8 @@ app:
                   arguments:
                     app:
                         help: App name
-                    panel:
-                        help: Select a specific panel
+                    key:
+                        help: Select a specific panel, section or a question
                         nargs: '?'
                     -f:
                         full: --full
diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index e1a96b866..0c3469c14 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -96,51 +96,72 @@ ynh_value_set() {
 
 _ynh_panel_get() {
     # From settings
-    local params_sources
-    params_sources=`python3 << EOL
+    local lines
+    lines=`python3 << EOL
 import toml
 from collections import OrderedDict
 with open("../config_panel.toml", "r") as f:
     file_content = f.read()
 loaded_toml = toml.loads(file_content, _dict=OrderedDict)
 
-for panel_name,panel in loaded_toml.items():
-    if isinstance(panel, dict):
-        for section_name, section in panel.items():
-            if isinstance(section, dict):
-                for name, param in section.items():
-                    if isinstance(param, dict) and param.get('type', 'string') not in ['success', 'info', 'warning', 'danger', 'display_text', 'markdown']:
-                        print("%s=%s" % (name, param.get('source', 'settings')))
+for panel_name, panel in loaded_toml.items():
+    if not isinstance(panel, dict): continue
+    for section_name, section in panel.items():
+        if not isinstance(section, dict): continue
+        for name, param in section.items():
+            if not isinstance(param, dict):
+                continue
+            print(';'.join([
+                name, 
+                param.get('type', 'string'),
+                param.get('source', 'settings' if param.get('type', 'string') != 'file' else '')
+                ]))
 EOL
 `
-    for param_source in $params_sources
+    for line in $lines
     do
-        local short_setting="$(echo $param_source | cut -d= -f1)"
+        IFS=';' read short_setting type source <<< "$line"
         local getter="get__${short_setting}"
-        local source="$(echo $param_source | cut -d= -f2)"
         sources[${short_setting}]="$source"
+        types[${short_setting}]="$type"
         file_hash[${short_setting}]=""
-
+        formats[${short_setting}]=""
         # Get value from getter if exists
         if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
             old[$short_setting]="$($getter)"
+            formats[${short_setting}]="yaml"
         
+        elif [[ "$source" == "" ]] ; then
+            old[$short_setting]="YNH_NULL"
+
         # Get value from app settings or from another file
-        elif [[ "$source" == "settings" ]] || [[ "$source" == *":"* ]] ; then
+        elif [[ "$type" == "file" ]] ; then
+            if [[ "$source" == "settings" ]] ; then
+                ynh_die "File '${short_setting}' can't be stored in settings"
+            fi
+            old[$short_setting]="$(ls $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)"
+            file_hash[$short_setting]="true"
+        
+        # Get multiline text from settings or from a full file
+        elif [[ "$type" == "text" ]] ; then
+            if [[ "$source" == "settings" ]] ; then
+                old[$short_setting]="$(ynh_app_setting_get $app $short_setting)"
+            elif [[ "$source" == *":"* ]] ; then
+                ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+            else
+                old[$short_setting]="$(cat $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)"
+            fi
+
+        # Get value from a kind of key/value file 
+        else
             if [[ "$source" == "settings" ]] ; then
                 source=":/etc/yunohost/apps/$app/settings.yml"
             fi
-        
             local source_key="$(echo "$source" | cut -d: -f1)"
             source_key=${source_key:-$short_setting}
             local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
             old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")"
         
-        # Specific case for files (all content of the file is the source)
-        else
-            
-            old[$short_setting]="$(ls $(echo $source | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)"
-            file_hash[$short_setting]="true"
         fi
     done
     
@@ -152,27 +173,42 @@ _ynh_panel_apply() {
     do
         local setter="set__${short_setting}"
         local source="${sources[$short_setting]}"
+        local type="${types[$short_setting]}"
         if [ "${changed[$short_setting]}" == "true" ] ; then 
             # Apply setter if exists
             if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
                 $setter
 
-            # Copy file in right place 
+            elif [[ "$source" == "" ]] ; then
+                continue
+
+            # Save in a file
+            elif [[ "$type" == "file" ]] ; then
+                if [[ "$source" == "settings" ]] ; then
+                    ynh_die "File '${short_setting}' can't be stored in settings"
+                fi
+                local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+                cp "${!short_setting}" "$source_file"
+            
+            # Save value in app settings 
             elif [[ "$source" == "settings" ]] ; then
                 ynh_app_setting_set $app $short_setting "${!short_setting}"
             
-            # Get value from a kind of key/value file
-            elif [[ "$source" == *":"* ]]
-            then
+            # Save multiline text in a file
+            elif [[ "$type" == "text" ]] ; then
+                if [[ "$source" == *":"* ]] ; then
+                    ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+                fi
+                local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+                echo "${!short_setting}" > "$source_file"
+
+            # Set value into a kind of key/value file
+            else
                 local source_key="$(echo "$source" | cut -d: -f1)"
                 source_key=${source_key:-$short_setting}
                 local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
                 ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}"
 
-            # Specific case for files (all content of the file is the source)
-            else
-                local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
-                cp "${!short_setting}" "$source_file"
             fi
         fi
     done
@@ -182,7 +218,13 @@ _ynh_panel_show() {
     for short_setting in "${!old[@]}"
     do
         if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then
-	        ynh_return "${short_setting}: \"${old[$short_setting]}\""
+            if [[ "${formats[$short_setting]}" == "yaml" ]] ; then
+                ynh_return "${short_setting}:"
+                ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/  /g')"
+            else
+                ynh_return "${short_setting}: \"$(echo "${old[$short_setting]}" | sed ':a;N;$!ba;s/\n/\n\n/g')\""
+
+            fi
         fi
     done
 }
@@ -225,7 +267,8 @@ _ynh_panel_validate() {
     done
     if [[ "$is_error" == "true" ]]
     then
-        ynh_die "Nothing has changed"
+        ynh_print_info "Nothing has changed"
+        exit 0
     fi 
     
     # Run validation if something is changed
@@ -241,7 +284,7 @@ _ynh_panel_validate() {
         if [ -n "$result" ]
         then
             local key="YNH_ERROR_${short_setting}"
-            ynh_return "$key: $result"
+            ynh_return "$key: \"$result\""
             is_error=true
         fi
     done
@@ -274,6 +317,9 @@ ynh_panel_run() {
     declare -Ag changed=()
     declare -Ag file_hash=()
     declare -Ag sources=()
+    declare -Ag types=()
+    declare -Ag formats=()
+    
     case $1 in
         show)
             ynh_panel_get
@@ -281,12 +327,12 @@ ynh_panel_run() {
             ;;
         apply) 
             max_progression=4
-            ynh_script_progression --message="Reading config panel description and current configuration..." --weight=1
+            ynh_script_progression --message="Reading config panel description and current configuration..."
             ynh_panel_get
             
             ynh_panel_validate
             
-            ynh_script_progression --message="Applying the new configuration..." --weight=1
+            ynh_script_progression --message="Applying the new configuration..."
             ynh_panel_apply
             ynh_script_progression --message="Configuration of $app completed" --last
             ;;
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 4e20c6950..2acdcb679 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -35,6 +35,7 @@ import glob
 import urllib.parse
 import base64
 import tempfile
+import readline
 from collections import OrderedDict
 
 from moulinette import msignals, m18n, msettings
@@ -1754,36 +1755,21 @@ def app_action_run(operation_logger, app, action, args=None):
 # * docstrings
 # * merge translations on the json once the workflow is in place
 @is_unit_operation()
-def app_config_show(operation_logger, app, panel='', full=False):
+def app_config_show(operation_logger, app, key='', full=False):
     # logger.warning(m18n.n("experimental_feature"))
 
     # Check app is installed
     _assert_is_installed(app)
 
-    panel = panel if panel else ''
-    operation_logger.start()
+    key = key if key else ''
 
     # Read config panel toml
-    config_panel = _get_app_config_panel(app, filter_key=panel)
+    config_panel = _get_app_hydrated_config_panel(operation_logger,
+                                                  app, filter_key=key)
 
     if not config_panel:
         return None
 
-    # Call config script to extract current values
-    parsed_values = _call_config_script(operation_logger, app, 'show')
-
-    # # Check and transform values if needed
-    # options = [option for _, _, option in _get_options_iterator(config_panel)]
-    # args_dict = _parse_args_in_yunohost_format(
-    #     parsed_values, options, False
-    # )
-
-    # Hydrate
-    logger.debug("Hydrating config with current value")
-    for _, _, option in _get_options_iterator(config_panel):
-        if option['name'] in parsed_values:
-            option["value"] = parsed_values[option['name']] #args_dict[option["name"]][0]
-
     # Format result in full or reduce mode
     if full:
         operation_logger.success()
@@ -1800,8 +1786,8 @@ def app_config_show(operation_logger, app, panel='', full=False):
         }
         if not option.get('optional', False):
             r_option['ask'] += ' *'
-        if option.get('value', None) is not None:
-            r_option['value'] = option['value']
+        if option.get('current_value', None) is not None:
+            r_option['value'] = option['current_value']
 
     operation_logger.success()
     return result
@@ -1812,7 +1798,6 @@ def app_config_get(operation_logger, app, key):
     # Check app is installed
     _assert_is_installed(app)
 
-    operation_logger.start()
 
     # Read config panel toml
     config_panel = _get_app_config_panel(app, filter_key=key)
@@ -1820,6 +1805,8 @@ def app_config_get(operation_logger, app, key):
     if not config_panel:
         raise YunohostError("app_config_no_panel")
 
+    operation_logger.start()
+
     # Call config script to extract current values
     parsed_values = _call_config_script(operation_logger, app, 'show')
 
@@ -1851,7 +1838,8 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
     filter_key = key if key else ''
 
     # Read config panel toml
-    config_panel = _get_app_config_panel(app, filter_key=filter_key)
+    config_panel = _get_app_hydrated_config_panel(operation_logger,
+                                                  app, filter_key=filter_key)
 
     if not config_panel:
         raise YunohostError("app_config_no_panel")
@@ -1862,12 +1850,16 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
     operation_logger.start()
 
     # Prepare pre answered questions
-    args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
+    if args:
+        args = { key: ','.join(value) for key, value in urllib.parse.parse_qs(args, keep_blank_values=True).items() }
+    else:
+        args = {}
     if value is not None:
         args = {filter_key.split('.')[-1]: value}
 
     try:
         logger.debug("Asking unanswered question and prevalidating...")
+        args_dict = {}
         for panel in config_panel.get("panel", []):
             if msettings.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
                 msignals.display(colorize("\n" + "=" * 40, 'purple'))
@@ -1878,13 +1870,13 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
                     msignals.display(colorize(f"\n# {section['name']}", 'purple'))
 
                 # Check and ask unanswered questions
-                args_dict = _parse_args_in_yunohost_format(
+                args_dict.update(_parse_args_in_yunohost_format(
                     args, section['options']
-                )
+                ))
 
         # Call config script to extract current values
         logger.info("Running config script...")
-        env = {key: str(value[0]) for key, value in args_dict.items()}
+        env = {key: str(value[0]) for key, value in args_dict.items() if not value[0] is None}
 
         errors = _call_config_script(operation_logger, app, 'apply', env=env)
     # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception
@@ -2246,6 +2238,37 @@ def _get_app_config_panel(app_id, filter_key=''):
 
     return None
 
+def _get_app_hydrated_config_panel(operation_logger, app, filter_key=''):
+
+    # Read config panel toml
+    config_panel = _get_app_config_panel(app, filter_key=filter_key)
+
+    if not config_panel:
+        return None
+
+    operation_logger.start()
+
+    # Call config script to extract current values
+    parsed_values = _call_config_script(operation_logger, app, 'show')
+
+    # # Check and transform values if needed
+    # options = [option for _, _, option in _get_options_iterator(config_panel)]
+    # args_dict = _parse_args_in_yunohost_format(
+    #     parsed_values, options, False
+    # )
+
+    # Hydrate
+    logger.debug("Hydrating config with current value")
+    for _, _, option in _get_options_iterator(config_panel):
+        if option['name'] in parsed_values:
+            value = parsed_values[option['name']]
+            if isinstance(value, dict):
+                option.update(value)
+            else:
+                option["current_value"] = value #args_dict[option["name"]][0]
+
+    return config_panel
+
 
 def _get_app_settings(app_id):
     """
@@ -2808,6 +2831,7 @@ class YunoHostArgumentFormatParser(object):
         parsed_question.name = question["name"]
         parsed_question.type = question.get("type", 'string')
         parsed_question.default = question.get("default", None)
+        parsed_question.current_value = question.get("current_value")
         parsed_question.optional = question.get("optional", False)
         parsed_question.choices = question.get("choices", [])
         parsed_question.pattern = question.get("pattern")
@@ -2835,11 +2859,20 @@ class YunoHostArgumentFormatParser(object):
                     msignals.display(text_for_user_input_in_cli)
 
                 elif question.value is None:
-                    question.value = msignals.prompt(
-                        message=text_for_user_input_in_cli,
-                        is_password=self.hide_user_input_in_prompt,
-                        confirm=self.hide_user_input_in_prompt
-                    )
+                    prefill = None
+                    if question.current_value is not None:
+                        prefill = question.current_value
+                    elif question.default is not None:
+                        prefill = question.default
+                    readline.set_startup_hook(lambda: readline.insert_text(prefill))
+                    try:
+                        question.value = msignals.prompt(
+                            message=text_for_user_input_in_cli,
+                            is_password=self.hide_user_input_in_prompt,
+                            confirm=self.hide_user_input_in_prompt
+                        )
+                    finally:
+                        readline.set_startup_hook()
 
 
             # Apply default value
@@ -2897,8 +2930,6 @@ class YunoHostArgumentFormatParser(object):
         if question.choices:
             text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices))
 
-        if question.default is not None:
-            text_for_user_input_in_cli += " (default: {0})".format(question.default)
         if question.help or question.helpLink:
             text_for_user_input_in_cli += ":\033[m"
         if question.help:
@@ -2919,6 +2950,18 @@ class StringArgumentParser(YunoHostArgumentFormatParser):
     default_value = ""
 
 
+class TagsArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "tags"
+
+    def _prevalidate(self, question):
+        values = question.value
+        for value in values.split(','):
+            question.value = value
+            super()._prevalidate(question)
+        question.value = values
+
+
+
 class PasswordArgumentParser(YunoHostArgumentFormatParser):
     hide_user_input_in_prompt = True
     argument_type = "password"
@@ -2938,13 +2981,15 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser):
         return question
 
     def _prevalidate(self, question):
-        if any(char in question.value for char in self.forbidden_chars):
-            raise YunohostValidationError(
-                "pattern_password_app", forbidden_chars=self.forbidden_chars
-            )
+        super()._prevalidate(question)
 
-        # If it's an optional argument the value should be empty or strong enough
-        if not question.optional or question.value:
+        if question.value is not None:
+            if any(char in question.value for char in self.forbidden_chars):
+                raise YunohostValidationError(
+                    "pattern_password_app", forbidden_chars=self.forbidden_chars
+                )
+
+            # If it's an optional argument the value should be empty or strong enough
             from yunohost.utils.password import assert_password_is_strong_enough
 
             assert_password_is_strong_enough("user", question.value)
@@ -3098,23 +3143,26 @@ class DisplayTextArgumentParser(YunoHostArgumentFormatParser):
     readonly = True
 
     def parse_question(self, question, user_answers):
-        question = super(DisplayTextArgumentParser, self).parse_question(
+        question_parsed = super().parse_question(
             question, user_answers
         )
 
-        question.optional = True
+        question_parsed.optional = True
+        question_parsed.style = question.get('style', 'info')
 
-        return question
+        return question_parsed
 
     def _format_text_for_user_input_in_cli(self, question):
         text = question.ask['en']
-        if question.type in ['info', 'warning', 'danger']:
+
+        if question.style in ['success', 'info', 'warning', 'danger']:
             color = {
+                'success': 'green',
                 'info': 'cyan',
                 'warning': 'yellow',
                 'danger': 'red'
             }
-            return colorize(m18n.g(question.type), color[question.type]) + f" {text}"
+            return colorize(m18n.g(question.style), color[question.style]) + f" {text}"
         else:
             return text
 
@@ -3137,7 +3185,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
         if question.get('accept'):
             question_parsed.accept = question.get('accept').replace(' ', '').split(',')
         else:
-            question.accept = []
+            question_parsed.accept = []
         if msettings.get('interface') == 'api':
             if user_answers.get(question_parsed.name):
                 question_parsed.value = {
@@ -3200,7 +3248,7 @@ ARGUMENTS_TYPE_PARSERS = {
     "string": StringArgumentParser,
     "text": StringArgumentParser,
     "select": StringArgumentParser,
-    "tags": StringArgumentParser,
+    "tags": TagsArgumentParser,
     "email": StringArgumentParser,
     "url": StringArgumentParser,
     "date": StringArgumentParser,
@@ -3214,10 +3262,7 @@ ARGUMENTS_TYPE_PARSERS = {
     "number": NumberArgumentParser,
     "range": NumberArgumentParser,
     "display_text": DisplayTextArgumentParser,
-    "success": DisplayTextArgumentParser,
-    "danger": DisplayTextArgumentParser,
-    "warning": DisplayTextArgumentParser,
-    "info": DisplayTextArgumentParser,
+    "alert": DisplayTextArgumentParser,
     "markdown": DisplayTextArgumentParser,
     "file": FileArgumentParser,
 }

From 766711069f674f7d48bc9d0ce406aae757cb8f7c Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 30 Aug 2021 13:25:32 +0200
Subject: [PATCH 38/51] [fix] Bad call to Moulinette.get

---
 src/yunohost/app.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 79944d340..f5e5dfd48 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1867,12 +1867,12 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
         logger.debug("Asking unanswered question and prevalidating...")
         args_dict = {}
         for panel in config_panel.get("panel", []):
-            if Moulinette.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
+            if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3:
                 Moulinette.display(colorize("\n" + "=" * 40, 'purple'))
                 Moulinette.display(colorize(f">>>> {panel['name']}", 'purple'))
                 Moulinette.display(colorize("=" * 40, 'purple'))
             for section in panel.get("sections", []):
-                if Moulinette.get('interface') == 'cli' and len(filter_key.split('.')) < 3:
+                if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3:
                     Moulinette.display(colorize(f"\n# {section['name']}", 'purple'))
 
                 # Check and ask unanswered questions
@@ -2855,9 +2855,10 @@ class YunoHostArgumentFormatParser(object):
     def parse(self, question, user_answers):
         question = self.parse_question(question, user_answers)
 
+<<<<<<< HEAD
         while True:
             # Display question if no value filled or if it's a readonly message
-            if Moulinette.get('interface') == 'cli':
+            if Moulinette.interface == 'cli':
                 text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
                     question
                 )
@@ -2893,7 +2894,7 @@ class YunoHostArgumentFormatParser(object):
             try:
                 self._prevalidate(question)
             except YunohostValidationError as e:
-                if Moulinette.get('interface') == 'api':
+                if Moulinette.interface == 'api':
                     raise
                 Moulinette.display(str(e), 'error')
                 question.value = None
@@ -3179,7 +3180,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
     @classmethod
     def clean_upload_dirs(cls):
         # Delete files uploaded from API
-        if Moulinette.get('interface') == 'api':
+        if Moulinette.interface == 'api':
             for upload_dir in cls.upload_dirs:
                 if os.path.exists(upload_dir):
                     shutil.rmtree(upload_dir)
@@ -3192,7 +3193,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
             question_parsed.accept = question.get('accept').replace(' ', '').split(',')
         else:
             question_parsed.accept = []
-        if Moulinette.get('interface') == 'api':
+        if Moulinette.interface == 'api':
             if user_answers.get(question_parsed.name):
                 question_parsed.value = {
                     'content': question_parsed.value,
@@ -3223,7 +3224,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
         if not question.value:
             return question.value
 
-        if Moulinette.get('interface') == 'api':
+        if Moulinette.interface == 'api':
 
             upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
             FileArgumentParser.upload_dirs += [upload_dir]

From 8d9f8c7123dbc8286025ae2b8acbafa8c03c746d Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 30 Aug 2021 14:04:12 +0200
Subject: [PATCH 39/51] [fix] Bad call to Moulinette.interface

---
 src/yunohost/app.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index f5e5dfd48..7ddbdc7f3 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1867,12 +1867,12 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
         logger.debug("Asking unanswered question and prevalidating...")
         args_dict = {}
         for panel in config_panel.get("panel", []):
-            if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3:
+            if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3:
                 Moulinette.display(colorize("\n" + "=" * 40, 'purple'))
                 Moulinette.display(colorize(f">>>> {panel['name']}", 'purple'))
                 Moulinette.display(colorize("=" * 40, 'purple'))
             for section in panel.get("sections", []):
-                if Moulinette.interface == 'cli' and len(filter_key.split('.')) < 3:
+                if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3:
                     Moulinette.display(colorize(f"\n# {section['name']}", 'purple'))
 
                 # Check and ask unanswered questions
@@ -2855,10 +2855,9 @@ class YunoHostArgumentFormatParser(object):
     def parse(self, question, user_answers):
         question = self.parse_question(question, user_answers)
 
-<<<<<<< HEAD
         while True:
             # Display question if no value filled or if it's a readonly message
-            if Moulinette.interface == 'cli':
+            if Moulinette.interface.type== 'cli':
                 text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
                     question
                 )
@@ -2894,7 +2893,7 @@ class YunoHostArgumentFormatParser(object):
             try:
                 self._prevalidate(question)
             except YunohostValidationError as e:
-                if Moulinette.interface == 'api':
+                if Moulinette.interface.type== 'api':
                     raise
                 Moulinette.display(str(e), 'error')
                 question.value = None
@@ -3180,7 +3179,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
     @classmethod
     def clean_upload_dirs(cls):
         # Delete files uploaded from API
-        if Moulinette.interface == 'api':
+        if Moulinette.interface.type== 'api':
             for upload_dir in cls.upload_dirs:
                 if os.path.exists(upload_dir):
                     shutil.rmtree(upload_dir)
@@ -3193,7 +3192,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
             question_parsed.accept = question.get('accept').replace(' ', '').split(',')
         else:
             question_parsed.accept = []
-        if Moulinette.interface == 'api':
+        if Moulinette.interface.type== 'api':
             if user_answers.get(question_parsed.name):
                 question_parsed.value = {
                     'content': question_parsed.value,
@@ -3224,7 +3223,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
         if not question.value:
             return question.value
 
-        if Moulinette.interface == 'api':
+        if Moulinette.interface.type== 'api':
 
             upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
             FileArgumentParser.upload_dirs += [upload_dir]

From 969564eec659d51cade553d4bfe092b2c4ba888c Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 30 Aug 2021 19:41:07 +0200
Subject: [PATCH 40/51] [fix] simple/double quotes into source

---
 data/helpers.d/configpanel | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index 0c3469c14..cfdbfc331 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -75,22 +75,21 @@ ynh_value_set() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
+    local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
     
-    local crazy_value="$(grep -i -o -P "${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
-    local var_part="^[ \t]*(\$?\w*\[)?[ \t]*[\"']?${key}[\"']?[ \t]*\]?[ \t]*[:=]>?[ \t]*"
+    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
     local first_char="${crazy_value:0:1}"
     if [[ "$first_char" == '"' ]] ; then
-        value="$(echo "$value" | sed 's/"/\\"/g')"
-        sed -ri "s%(${var_part}\")[^\"]*(\"[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+        value="$(echo "$value" | sed 's/"/\"/g')"
+        sed -ri 's%^('"${var_part}"'")[^"]*("[ \t;,]*)$%\1'"${value}"'\3%i' ${file}
     elif [[ "$first_char" == "'" ]] ; then
-        value="$(echo "$value" | sed "s/'/\\\\'/g")"
-        sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+        value="$(echo "$value" | sed "s/'/"'\'"'/g")"
+        sed -ri "s%^(${var_part}')[^']*('"'[ \t,;]*)$%\1'"${value}"'\3%i' ${file}
     else
         if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
-            value="\"$(echo "$value" | sed 's/"/\\"/g')\""
+            value='\"'"$(echo "$value" | sed 's/"/\"/g')"'\"'
         fi
-        sed -ri "s%(${var_part}')[^']*('[ \t\n,;]*)\$%\1${value}\2%i" ${file}
+        sed -ri "s%^(${var_part}).*"'$%\1'"${value}"'%i' ${file}
     fi
 }
 
@@ -222,7 +221,7 @@ _ynh_panel_show() {
                 ynh_return "${short_setting}:"
                 ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/  /g')"
             else
-                ynh_return "${short_setting}: \"$(echo "${old[$short_setting]}" | sed ':a;N;$!ba;s/\n/\n\n/g')\""
+                ynh_return "${short_setting}: "'"'"$(echo "${old[$short_setting]}" | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\n\n/g')"'"'
 
             fi
         fi

From c20226fc54f3e1772e7231907796e076074b7198 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Mon, 30 Aug 2021 19:41:30 +0200
Subject: [PATCH 41/51] [enh] Move prefill feature into moulinette

---
 src/yunohost/app.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 7ddbdc7f3..2ea1a3b70 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -35,7 +35,6 @@ import glob
 import urllib.parse
 import base64
 import tempfile
-import readline
 from collections import OrderedDict
 
 from moulinette.interfaces.cli import colorize
@@ -2865,20 +2864,18 @@ class YunoHostArgumentFormatParser(object):
                     Moulinette.display(text_for_user_input_in_cli)
 
                 elif question.value is None:
-                    prefill = None
+                    prefill = ""
                     if question.current_value is not None:
                         prefill = question.current_value
                     elif question.default is not None:
                         prefill = question.default
-                    readline.set_startup_hook(lambda: readline.insert_text(prefill))
-                    try:
-                        question.value = Moulinette.prompt(
-                            message=text_for_user_input_in_cli,
-                            is_password=self.hide_user_input_in_prompt,
-                            confirm=self.hide_user_input_in_prompt
-                        )
-                    finally:
-                        readline.set_startup_hook()
+                    question.value = Moulinette.prompt(
+                        message=text_for_user_input_in_cli,
+                        is_password=self.hide_user_input_in_prompt,
+                        confirm=self.hide_user_input_in_prompt,
+                        prefill=prefill,
+                        is_multiline=(question.type == "text")
+                    )
 
 
             # Apply default value

From bb11b5dcacc6ad8f6dfeb9ccb9e98a4bca7a39bd Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 31 Aug 2021 04:20:21 +0200
Subject: [PATCH 42/51] [enh] Be able to delete source file

---
 data/helpers.d/configpanel | 10 +++++++++-
 src/yunohost/app.py        | 10 +++++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index cfdbfc331..e99a66d4c 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -187,7 +187,11 @@ _ynh_panel_apply() {
                     ynh_die "File '${short_setting}' can't be stored in settings"
                 fi
                 local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
-                cp "${!short_setting}" "$source_file"
+                if [[ "${!short_setting}" == "" ]] ; then
+                    rm -f "$source_file"
+                else
+                    cp "${!short_setting}" "$source_file"
+                fi
             
             # Save value in app settings 
             elif [[ "$source" == "settings" ]] ; then
@@ -247,6 +251,10 @@ _ynh_panel_validate() {
             file_hash[new__$short_setting]=""
             if [ -f "${old[$short_setting]}" ] ; then
                 file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
+                if [ -z "${!short_setting}" ] ; then
+                    changed[$short_setting]=true
+                    is_error=false
+                fi
             fi
             if [ -f "${!short_setting}" ] ; then
                 file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1)
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 2ea1a3b70..425f04023 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -3190,20 +3190,24 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
         else:
             question_parsed.accept = []
         if Moulinette.interface.type== 'api':
-            if user_answers.get(question_parsed.name):
+            if user_answers.get(f"{question_parsed.name}[name]"):
                 question_parsed.value = {
                     'content': question_parsed.value,
                     'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name),
                 }
+        # If path file are the same
+        if question_parsed.value and str(question_parsed.value) == question_parsed.current_value:
+            question_parsed.value = None
+
         return question_parsed
 
     def _prevalidate(self, question):
         super()._prevalidate(question)
-        if isinstance(question.value, str) and not os.path.exists(question.value):
+        if isinstance(question.value, str) and question.value and not os.path.exists(question.value):
             raise YunohostValidationError(
                 "app_argument_invalid", name=question.name, error=m18n.n("invalid_number1")
             )
-        if question.value is None or not question.accept:
+        if question.value in [None, ''] or not question.accept:
             return
 
         filename = question.value if isinstance(question.value, str) else question.value['filename']

From 08e7fcc48ef94aa48bc328878f06bb10d81af82d Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 31 Aug 2021 17:44:42 +0200
Subject: [PATCH 43/51] [enh] Be able to correctly display the error

---
 src/yunohost/utils/error.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/utils/error.py b/src/yunohost/utils/error.py
index f9b4ac61a..8405830e7 100644
--- a/src/yunohost/utils/error.py
+++ b/src/yunohost/utils/error.py
@@ -59,4 +59,4 @@ class YunohostValidationError(YunohostError):
 
     def content(self):
 
-        return {"error": self.strerror, "error_key": self.key}
+        return {"error": self.strerror, "error_key": self.key, **self.kwargs}

From 6d16e22f8779c2d7741c6e6b20c5ef301ac11d57 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Tue, 31 Aug 2021 17:45:13 +0200
Subject: [PATCH 44/51] [enh] VisibleIf on config panel section

---
 src/yunohost/app.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 425f04023..69c65046a 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -2206,9 +2206,11 @@ def _get_app_config_panel(app_id, filter_key=''):
                     "id": section_key,
                     "name": section_value.get("name", ""),
                     "optional": section_value.get("optional", True),
-                    "services": value.get("services", []),
+                    "services": section_value.get("services", []),
                     "options": [],
                 }
+                if section_value.get('visibleIf'):
+                    section['visibleIf'] = section_value.get('visibleIf')
 
                 options = [
                     k_v
@@ -2952,6 +2954,11 @@ class StringArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "string"
     default_value = ""
 
+    def _prevalidate(self, question):
+        super()._prevalidate(question)
+        raise YunohostValidationError(
+            "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2")
+        )
 
 class TagsArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "tags"
@@ -3065,7 +3072,7 @@ class DomainArgumentParser(YunoHostArgumentFormatParser):
 
     def _raise_invalid_answer(self, question):
         raise YunohostValidationError(
-            "app_argument_invalid", name=question.name, error=m18n.n("domain_unknown")
+            "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown")
         )
 
 
@@ -3092,7 +3099,7 @@ class UserArgumentParser(YunoHostArgumentFormatParser):
     def _raise_invalid_answer(self, question):
         raise YunohostValidationError(
             "app_argument_invalid",
-            name=question.name,
+            field=question.name,
             error=m18n.n("user_unknown", user=question.value),
         )
 
@@ -3116,17 +3123,17 @@ class NumberArgumentParser(YunoHostArgumentFormatParser):
         super()._prevalidate(question)
         if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()):
             raise YunohostValidationError(
-                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
             )
 
         if question.min is not None and int(question.value) < question.min:
             raise YunohostValidationError(
-                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
             )
 
         if question.max is not None and int(question.value) > question.max:
             raise YunohostValidationError(
-                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
             )
 
     def _post_parse_value(self, question):
@@ -3137,7 +3144,7 @@ class NumberArgumentParser(YunoHostArgumentFormatParser):
             return int(question.value)
 
         raise YunohostValidationError(
-            "app_argument_invalid", name=question.name, error=m18n.n("invalid_number")
+            "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
         )
 
 
@@ -3205,7 +3212,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
         super()._prevalidate(question)
         if isinstance(question.value, str) and question.value and not os.path.exists(question.value):
             raise YunohostValidationError(
-                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number1")
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1")
             )
         if question.value in [None, ''] or not question.accept:
             return
@@ -3213,7 +3220,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
         filename = question.value if isinstance(question.value, str) else question.value['filename']
         if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept:
             raise YunohostValidationError(
-                "app_argument_invalid", name=question.name, error=m18n.n("invalid_number2")
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2")
             )
 
 

From 7d26b1477fe12756b798defcec0d3126e3a2368f Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Thu, 2 Sep 2021 02:27:22 +0200
Subject: [PATCH 45/51] [enh] Some refactoring for config panel

---
 data/actionsmap/yunohost.yml |  36 ++-
 data/helpers.d/configpanel   |  32 ++-
 locales/en.json              |   2 +-
 src/yunohost/app.py          | 456 +++++++++++++++++------------------
 4 files changed, 263 insertions(+), 263 deletions(-)

diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml
index 2c91651bd..adc6aa93a 100644
--- a/data/actionsmap/yunohost.yml
+++ b/data/actionsmap/yunohost.yml
@@ -831,34 +831,28 @@ app:
           subcategory_help: Applications configuration panel
           actions:
 
-              ### app_config_show()
-              show:
-                  action_help: show config panel for the application
+              ### app_config_get()
+              get:
+                  action_help: Display an app configuration
                   api: GET /apps/<app>/config-panel
                   arguments:
                     app:
                         help: App name
                     key:
-                        help: Select a specific panel, section or a question
+                        help: A specific panel, section or a question identifier
                         nargs: '?'
-                    -f:
-                        full: --full
-                        help: Display all info known about the config-panel.
-                        action: store_true
-
-              ### app_config_get()
-              get:
-                  action_help: show config panel for the application
-                  api: GET /apps/<app>/config-panel/<key>
-                  arguments:
-                    app:
-                        help: App name
-                    key:
-                        help: The question identifier 
+                    -m:
+                        full: --mode
+                        help: Display mode to use
+                        choices:
+                            - classic
+                            - full
+                            - export
+                        default: classic
 
               ### app_config_set()
               set:
-                  action_help: apply the new configuration
+                  action_help: Apply a new configuration
                   api: PUT /apps/<app>/config
                   arguments:
                     app:
@@ -872,6 +866,10 @@ app:
                     -a:
                         full: --args
                         help: Serialized arguments for new configuration (i.e. "domain=domain.tld&path=/path")
+                    -f:
+                        full: --args-file
+                        help: YAML or JSON file with key/value couples
+                        type: open
 
 #############################
 #          Backup           #
diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index e99a66d4c..b55b0b0fd 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -74,23 +74,29 @@ ynh_value_set() {
     local value
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-
     local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
     
-    local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)"
+    # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
     local first_char="${crazy_value:0:1}"
     if [[ "$first_char" == '"' ]] ; then
-        value="$(echo "$value" | sed 's/"/\"/g')"
-        sed -ri 's%^('"${var_part}"'")[^"]*("[ \t;,]*)$%\1'"${value}"'\3%i' ${file}
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # So we need \\\\ to go through 2 sed
+        value="$(echo "$value" | sed 's/"/\\\\"/g')"
+        sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file}
     elif [[ "$first_char" == "'" ]] ; then
-        value="$(echo "$value" | sed "s/'/"'\'"'/g")"
-        sed -ri "s%^(${var_part}')[^']*('"'[ \t,;]*)$%\1'"${value}"'\3%i' ${file}
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # However double quotes implies to double \\ to
+        # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str
+        value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")"
+        sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file}
     else
         if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
-            value='\"'"$(echo "$value" | sed 's/"/\"/g')"'\"'
+            value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"'
         fi
-        sed -ri "s%^(${var_part}).*"'$%\1'"${value}"'%i' ${file}
+        sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file}
     fi
+    ynh_print_info "Configuration key '$key' edited into $file"
 }
 
 _ynh_panel_get() {
@@ -189,13 +195,16 @@ _ynh_panel_apply() {
                 local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
                 if [[ "${!short_setting}" == "" ]] ; then
                     rm -f "$source_file"
+                    ynh_print_info "File '$source_file' removed"
                 else
                     cp "${!short_setting}" "$source_file"
+                    ynh_print_info "File '$source_file' overwrited with ${!short_setting}"
                 fi
             
             # Save value in app settings 
             elif [[ "$source" == "settings" ]] ; then
                 ynh_app_setting_set $app $short_setting "${!short_setting}"
+                ynh_print_info "Configuration key '$short_setting' edited in app settings"
             
             # Save multiline text in a file
             elif [[ "$type" == "text" ]] ; then
@@ -204,13 +213,20 @@ _ynh_panel_apply() {
                 fi
                 local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
                 echo "${!short_setting}" > "$source_file"
+                ynh_print_info "File '$source_file' overwrited with the content you provieded in '${short_setting}' question"
 
             # Set value into a kind of key/value file
             else
                 local source_key="$(echo "$source" | cut -d: -f1)"
                 source_key=${source_key:-$short_setting}
                 local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+                
+                ynh_backup_if_checksum_is_different --file="$source_file"
                 ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}"
+                ynh_store_file_checksum --file="$source_file"
+                
+                # We stored the info in settings in order to be able to upgrade the app
+                ynh_app_setting_set $app $short_setting "${!short_setting}"
 
             fi
         fi
diff --git a/locales/en.json b/locales/en.json
index 3498c00e7..dcbf0f866 100644
--- a/locales/en.json
+++ b/locales/en.json
@@ -14,7 +14,7 @@
     "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.",
     "app_already_up_to_date": "{app} is already up-to-date",
     "app_argument_choice_invalid": "Use one of these choices '{choices}' for the argument '{name}' instead of '{value}'",
-    "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}",
+    "app_argument_invalid": "Pick a valid value for the argument '{field}': {error}",
     "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reason",
     "app_argument_required": "Argument '{name}' is required",
     "app_change_url_failed_nginx_reload": "Could not reload NGINX. Here is the output of 'nginx -t':\n{nginx_errors}",
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 69c65046a..a4c215328 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1756,135 +1756,122 @@ def app_action_run(operation_logger, app, action, args=None):
     return logger.success("Action successed!")
 
 
-# Config panel todo list:
-# * docstrings
-# * merge translations on the json once the workflow is in place
 @is_unit_operation()
-def app_config_show(operation_logger, app, key='', full=False):
-    # logger.warning(m18n.n("experimental_feature"))
+def app_config_get(operation_logger, app, key='', mode='classic'):
+    """
+    Display an app configuration in classic, full or export mode
+    """
 
     # Check app is installed
     _assert_is_installed(app)
 
-    key = key if key else ''
+    filter_key = key or ''
 
     # Read config panel toml
-    config_panel = _get_app_hydrated_config_panel(operation_logger,
-                                                  app, filter_key=key)
+    config_panel = _get_app_config_panel(app, filter_key=filter_key)
 
     if not config_panel:
-        return None
+        raise YunohostError("app_config_no_panel")
 
-    # Format result in full or reduce mode
-    if full:
+    # Call config script in order to hydrate config panel with current values
+    values = _call_config_script(operation_logger, app, 'show', config_panel=config_panel)
+
+    # Format result in full mode
+    if mode == 'full':
         operation_logger.success()
         return config_panel
 
-    result = OrderedDict()
-    for panel, section, option in _get_options_iterator(config_panel):
-        if panel['id'] not in result:
-            r_panel = result[panel['id']] = OrderedDict()
-        if section['id'] not in r_panel:
-            r_section = r_panel[section['id']] = OrderedDict()
-        r_option = r_section[option['name']] = {
-            "ask": option['ask']['en']
-        }
-        if not option.get('optional', False):
-            r_option['ask'] += ' *'
-        if option.get('current_value', None) is not None:
-            r_option['value'] = option['current_value']
+    # In 'classic' mode, we display the current value if key refer to an option
+    if filter_key.count('.') == 2 and mode == 'classic':
+        option = filter_key.split('.')[-1]
+        operation_logger.success()
+        return values.get(option, None)
+
+    # Format result in 'classic' or 'export' mode
+    logger.debug(f"Formating result in '{mode}' mode")
+    result = {}
+    for panel, section, option in _get_config_iterator(config_panel):
+        key = f"{panel['id']}.{section['id']}.{option['id']}"
+        if mode == 'export':
+            result[option['id']] = option.get('current_value')
+        else:
+            result[key] = { 'ask': _value_for_locale(option['ask']) }
+            if 'current_value' in option:
+                result[key]['value'] = option['current_value']
 
     operation_logger.success()
     return result
 
 
 @is_unit_operation()
-def app_config_get(operation_logger, app, key):
+def app_config_set(operation_logger, app, key=None, value=None, args=None, args_file=None):
+    """
+    Apply a new app configuration
+    """
+
     # Check app is installed
     _assert_is_installed(app)
 
+    filter_key = key or ''
 
     # Read config panel toml
-    config_panel = _get_app_config_panel(app, filter_key=key)
+    config_panel = _get_app_config_panel(app, filter_key=filter_key)
 
     if not config_panel:
         raise YunohostError("app_config_no_panel")
 
-    operation_logger.start()
-
-    # Call config script to extract current values
-    parsed_values = _call_config_script(operation_logger, app, 'show')
-
-    logger.debug("Searching value")
-    short_key = key.split('.')[-1]
-    if short_key not in parsed_values:
-        return None
-
-    return parsed_values[short_key]
-
-    # for panel, section, option in _get_options_iterator(config_panel):
-    #     if option['name'] == short_key:
-    #         # Check and transform values if needed
-    #         args_dict = _parse_args_in_yunohost_format(
-    #             parsed_values, [option], False
-    #         )
-    #         operation_logger.success()
-
-    #         return args_dict[short_key][0]
-
-    # return None
-
-
-@is_unit_operation()
-def app_config_set(operation_logger, app, key=None, value=None, args=None):
-    # Check app is installed
-    _assert_is_installed(app)
-
-    filter_key = key if key else ''
-
-    # Read config panel toml
-    config_panel = _get_app_hydrated_config_panel(operation_logger,
-                                                  app, filter_key=filter_key)
-
-    if not config_panel:
-        raise YunohostError("app_config_no_panel")
-
-    if args is not None and value is not None:
+    if (args is not None or args_file is not None) and value is not None:
         raise YunohostError("app_config_args_value")
 
-    operation_logger.start()
+    if filter_key.count('.') != 2 and not value is None:
+        raise YunohostError("app_config_set_value_on_section")
+
+    # Import and parse pre-answered options
+    logger.debug("Import and parse pre-answered options")
+    args = urllib.parse.parse_qs(args or '', keep_blank_values=True)
+    args = { key: ','.join(value_) for key, value_ in args.items() }
+
+    if args_file:
+        # Import YAML / JSON file but keep --args values
+        args = { **read_yaml(args_file), **args }
 
-    # Prepare pre answered questions
-    if args:
-        args = { key: ','.join(value) for key, value in urllib.parse.parse_qs(args, keep_blank_values=True).items() }
-    else:
-        args = {}
     if value is not None:
         args = {filter_key.split('.')[-1]: value}
 
+    # Call config script in order to hydrate config panel with current values
+    _call_config_script(operation_logger, app, 'show', config_panel=config_panel)
+
+    # Ask unanswered question and prevalidate
+    logger.debug("Ask unanswered question and prevalidate data")
+    def display_header(message):
+        """ CLI panel/section header display
+        """
+        if Moulinette.interface.type == 'cli' and filter_key.count('.') < 2:
+            Moulinette.display(colorize(message, 'purple'))
+
     try:
-        logger.debug("Asking unanswered question and prevalidating...")
-        args_dict = {}
-        for panel in config_panel.get("panel", []):
-            if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3:
-                Moulinette.display(colorize("\n" + "=" * 40, 'purple'))
-                Moulinette.display(colorize(f">>>> {panel['name']}", 'purple'))
-                Moulinette.display(colorize("=" * 40, 'purple'))
-            for section in panel.get("sections", []):
-                if Moulinette.interface.type== 'cli' and len(filter_key.split('.')) < 3:
-                    Moulinette.display(colorize(f"\n# {section['name']}", 'purple'))
+        env = {}
+        for panel, section, obj in _get_config_iterator(config_panel,
+                                                    ['panel', 'section']):
+            if panel == obj:
+                name = _value_for_locale(panel['name'])
+                display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
+                continue
+            name = _value_for_locale(section['name'])
+            display_header(f"\n# {name}")
 
-                # Check and ask unanswered questions
-                args_dict.update(_parse_args_in_yunohost_format(
-                    args, section['options']
-                ))
+            # Check and ask unanswered questions
+            env.update(_parse_args_in_yunohost_format(
+                args, section['options']
+            ))
 
-        # Call config script to extract current values
+        # Call config script in 'apply' mode
         logger.info("Running config script...")
-        env = {key: str(value[0]) for key, value in args_dict.items() if not value[0] is None}
+        env = {key: str(value[0]) for key, value in env.items() if not value[0] is None}
 
         errors = _call_config_script(operation_logger, app, 'apply', env=env)
-    # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception
+    # Script got manually interrupted ...
+    # N.B. : KeyboardInterrupt does not inherit from Exception
     except (KeyboardInterrupt, EOFError):
         error = m18n.n("operation_interrupted")
         logger.error(m18n.n("app_config_failed", app=app, error=error))
@@ -1904,25 +1891,20 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
 
     if errors:
         return {
-            "app": app,
             "errors": errors,
         }
 
     # Reload services
     logger.info("Reloading services...")
-    services_to_reload = set([])
-    for panel in config_panel.get("panel", []):
-        services_to_reload |= set(panel.get('services', []))
-        for section in panel.get("sections", []):
-            services_to_reload |= set(section.get('services', []))
-            for option in section.get("options", []):
-                services_to_reload |= set(option.get('services', []))
+    services_to_reload = set()
+    for panel, section, obj in _get_config_iterator(config_panel,
+                                                    ['panel', 'section', 'option']):
+        services_to_reload |= set(obj.get('services', []))
 
     services_to_reload = list(services_to_reload)
     services_to_reload.sort(key = 'nginx'.__eq__)
     for service in services_to_reload:
-        if service == "__APP__":
-            service = app
+        service = service.replace('__APP__', app)
         logger.debug(f"Reloading {service}")
         if not _run_service_command('reload-or-restart', service):
             services = _get_services()
@@ -1934,23 +1916,27 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None):
             )
 
     logger.success("Config updated as expected")
-    return {
-        "app": app,
-        "errors": [],
-        "logs": operation_logger.success(),
-    }
+    return {}
 
 
-def _get_options_iterator(config_panel):
-    for panel in config_panel.get("panel", []):
+def _get_config_iterator(config_panel, trigger=['option']):
+    for panel in config_panel.get("panels", []):
+        if 'panel' in trigger:
+            yield (panel, None, panel)
         for section in panel.get("sections", []):
-            for option in section.get("options", []):
-                yield (panel, section, option)
+            if 'section' in trigger:
+                yield (panel, section, section)
+            if 'option' in trigger:
+                for option in section.get("options", []):
+                    yield (panel, section, option)
 
 
-def _call_config_script(operation_logger, app, action, env={}):
+def _call_config_script(operation_logger, app, action, env={}, config_panel=None):
     from yunohost.hook import hook_exec
 
+    YunoHostArgumentFormatParser.operation_logger = operation_logger
+    operation_logger.start()
+
     # Add default config script if needed
     config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config")
     if not os.path.exists(config_script):
@@ -1976,7 +1962,27 @@ ynh_panel_run $1
         config_script, args=[action], env=env
     )
     if ret != 0:
-        operation_logger.error(parsed_values)
+        if action == 'show':
+            raise YunohostError("app_config_unable_to_read_values")
+        else:
+            raise YunohostError("app_config_unable_to_apply_values_correctly")
+
+        return parsed_values
+
+    if not config_panel:
+        return parsed_values
+
+    # Hydrating config panel with current value
+    logger.debug("Hydrating config with current values")
+    for _, _, option in _get_config_iterator(config_panel):
+        if option['name'] not in parsed_values:
+            continue
+        value = parsed_values[option['name']]
+        # In general, the value is just a simple value.
+        # Sometimes it could be a dict used to overwrite the option itself
+        value = value if isinstance(value, dict) else {'current_value': value }
+        option.update(value)
+
     return parsed_values
 
 
@@ -2083,6 +2089,13 @@ def _get_app_actions(app_id):
 
 def _get_app_config_panel(app_id, filter_key=''):
     "Get app config panel stored in json or in toml"
+
+    # Split filter_key
+    filter_key = dict(enumerate(filter_key.split('.')))
+    if len(filter_key) > 3:
+        raise YunohostError("app_config_too_much_sub_keys")
+
+    # Open TOML
     config_panel_toml_path = os.path.join(
         APPS_SETTING_PATH, app_id, "config_panel.toml"
     )
@@ -2103,7 +2116,7 @@ def _get_app_config_panel(app_id, filter_key=''):
     #         name = "Choose the sources of packages to automatically upgrade."
     #         default = "Security only"
     #         type = "text"
-    #         help = "We can't use a choices field for now. In the meantime please choose between one of this values:<br>Security only, Security and updates."
+    #         help = "We can't use a choices field for now. In the meantime[...]"
     #         # choices = ["Security only", "Security and updates"]
 
     #         [main.unattended_configuration.ynh_update]
@@ -2143,7 +2156,7 @@ def _get_app_config_panel(app_id, filter_key=''):
     #      u'name': u'50unattended-upgrades configuration file',
     #      u'options': [{u'//': u'"choices" : ["Security only", "Security and updates"]',
     #        u'default': u'Security only',
-    #        u'help': u"We can't use a choices field for now. In the meantime please choose between one of this values:<br>Security only, Security and updates.",
+    #        u'help': u"We can't use a choices field for now. In the meantime[...]",
     #        u'id': u'upgrade_level',
     #        u'name': u'Choose the sources of packages to automatically upgrade.',
     #        u'type': u'text'},
@@ -2152,127 +2165,81 @@ def _get_app_config_panel(app_id, filter_key=''):
     #        u'name': u'Would you like to update YunoHost packages automatically ?',
     #        u'type': u'bool'},
 
-    if os.path.exists(config_panel_toml_path):
-        toml_config_panel = toml.load(
-            open(config_panel_toml_path, "r"), _dict=OrderedDict
-        )
-        if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED:
-            raise YunohostError(
-                "app_config_too_old_version", app=app_id,
-                version=toml_config_panel["version"]
-            )
-
-        # transform toml format into json format
-        config_panel = {
-            "name": toml_config_panel["name"],
-            "version": toml_config_panel["version"],
-            "panel": [],
-        }
-        filter_key = filter_key.split('.')
-        filter_panel = filter_key.pop(0)
-        filter_section = filter_key.pop(0) if len(filter_key) > 0 else False
-        filter_option = filter_key.pop(0) if len(filter_key) > 0 else False
-
-        panels = [
-            key_value
-            for key_value in toml_config_panel.items()
-            if key_value[0] not in ("name", "version")
-            and isinstance(key_value[1], OrderedDict)
-        ]
-
-        for key, value in panels:
-            if filter_panel and key != filter_panel:
-                continue
-
-            panel = {
-                "id": key,
-                "name": value.get("name", ""),
-                "services": value.get("services", []),
-                "sections": [],
-            }
-
-            sections = [
-                k_v1
-                for k_v1 in value.items()
-                if k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict)
-            ]
-
-            for section_key, section_value in sections:
-
-                if filter_section and section_key != filter_section:
-                    continue
-
-                section = {
-                    "id": section_key,
-                    "name": section_value.get("name", ""),
-                    "optional": section_value.get("optional", True),
-                    "services": section_value.get("services", []),
-                    "options": [],
-                }
-                if section_value.get('visibleIf'):
-                    section['visibleIf'] = section_value.get('visibleIf')
-
-                options = [
-                    k_v
-                    for k_v in section_value.items()
-                    if k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict)
-                ]
-
-                for option_key, option_value in options:
-                    if filter_option and option_key != filter_option:
-                        continue
-
-                    option = dict(option_value)
-                    option["optional"] = option_value.get("optional", section['optional'])
-                    option["name"] = option_key
-                    option["ask"] = {"en": option["ask"]}
-                    if "help" in option:
-                        option["help"] = {"en": option["help"]}
-                    section["options"].append(option)
-
-                panel["sections"].append(section)
-
-            config_panel["panel"].append(panel)
-
-        if (filter_panel and len(config_panel['panel']) == 0) or \
-           (filter_section and len(config_panel['panel'][0]['sections']) == 0) or \
-           (filter_option and len(config_panel['panel'][0]['sections'][0]['options']) == 0):
-            raise YunohostError(
-                "app_config_bad_filter_key", app=app_id, filter_key=filter_key
-            )
-
-        return config_panel
-
-    return None
-
-def _get_app_hydrated_config_panel(operation_logger, app, filter_key=''):
-
-    # Read config panel toml
-    config_panel = _get_app_config_panel(app, filter_key=filter_key)
-
-    if not config_panel:
+    if not os.path.exists(config_panel_toml_path):
         return None
+    toml_config_panel = read_toml(config_panel_toml_path)
 
-    operation_logger.start()
+    # Check TOML config panel is in a supported version
+    if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED:
+        raise YunohostError(
+            "app_config_too_old_version", app=app_id,
+            version=toml_config_panel["version"]
+        )
 
-    # Call config script to extract current values
-    parsed_values = _call_config_script(operation_logger, app, 'show')
+    # Transform toml format into internal format
+    defaults = {
+        'toml': {
+            'version': 1.0
+        },
+        'panels': {
+            'name': '',
+            'services': [],
+            'actions': {'apply': {'en': 'Apply'}}
+        }, # help
+        'sections': {
+            'name': '',
+            'services': [],
+            'optional': True
+        }, # visibleIf help
+        'options': {}
+        # ask type source help helpLink example style icon placeholder visibleIf
+        # optional choices pattern limit min max step accept redact
+    }
 
-    # # Check and transform values if needed
-    # options = [option for _, _, option in _get_options_iterator(config_panel)]
-    # args_dict = _parse_args_in_yunohost_format(
-    #     parsed_values, options, False
-    # )
+    def convert(toml_node, node_type):
+        """Convert TOML in internal format ('full' mode used by webadmin)
 
-    # Hydrate
-    logger.debug("Hydrating config with current value")
-    for _, _, option in _get_options_iterator(config_panel):
-        if option['name'] in parsed_values:
-            value = parsed_values[option['name']]
-            if isinstance(value, dict):
-                option.update(value)
+        Here are some properties of 1.0 config panel in toml:
+          - node properties and node children are mixed,
+          - text are in english only
+          - some properties have default values
+        This function detects all children nodes and put them in a list
+        """
+        # Prefill the node default keys if needed
+        default = defaults[node_type]
+        node = {key: toml_node.get(key, value) for key, value in default.items()}
+
+        # Define the filter_key part to use and the children type
+        i = list(defaults).index(node_type)
+        search_key = filter_key.get(i)
+        subnode_type = list(defaults)[i+1] if node_type != 'options' else None
+
+        for key, value in toml_node.items():
+            # Key/value are a child node
+            if isinstance(value, OrderedDict) and key not in default and subnode_type:
+                # We exclude all nodes not referenced by the filter_key
+                if search_key and key != search_key:
+                    continue
+                subnode = convert(value, subnode_type)
+                subnode['id'] = key
+                if node_type == 'sections':
+                    subnode['name'] = key # legacy
+                    subnode.setdefault('optional', toml_node.get('optional', True))
+                node.setdefault(subnode_type, []).append(subnode)
+            # Key/value are a property
             else:
-                option["current_value"] = value #args_dict[option["name"]][0]
+                # Todo search all i18n keys
+                node[key] = value if key not in ['ask', 'help', 'name'] else { 'en': value }
+        return node
+
+    config_panel = convert(toml_config_panel, 'toml')
+
+    try:
+        config_panel['panels'][0]['sections'][0]['options'][0]
+    except (KeyError, IndexError):
+        raise YunohostError(
+            "app_config_empty_or_bad_filter_key", app=app_id, filter_key=filter_key
+        )
 
     return config_panel
 
@@ -2831,6 +2798,7 @@ class Question:
 
 class YunoHostArgumentFormatParser(object):
     hide_user_input_in_prompt = False
+    operation_logger = None
 
     def parse_question(self, question, user_answers):
         parsed_question = Question()
@@ -2842,10 +2810,11 @@ class YunoHostArgumentFormatParser(object):
         parsed_question.optional = question.get("optional", False)
         parsed_question.choices = question.get("choices", [])
         parsed_question.pattern = question.get("pattern")
-        parsed_question.ask = question.get("ask", {'en': f"Enter value for '{parsed_question.name}':"})
+        parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"})
         parsed_question.help = question.get("help")
         parsed_question.helpLink = question.get("helpLink")
         parsed_question.value = user_answers.get(parsed_question.name)
+        parsed_question.redact = question.get('redact', False)
 
         # Empty value is parsed as empty string
         if parsed_question.default == "":
@@ -2947,6 +2916,27 @@ class YunoHostArgumentFormatParser(object):
         return text_for_user_input_in_cli
 
     def _post_parse_value(self, question):
+        if not question.redact:
+            return question.value
+
+        # Tell the operation_logger to redact all password-type / secret args
+        # Also redact the % escaped version of the password that might appear in
+        # the 'args' section of metadata (relevant for password with non-alphanumeric char)
+        data_to_redact = []
+        if question.value and isinstance(question.value, str):
+            data_to_redact.append(question.value)
+        if question.current_value and isinstance(question.current_value, str):
+            data_to_redact.append(question.current_value)
+        data_to_redact += [
+            urllib.parse.quote(data)
+            for data in data_to_redact
+            if urllib.parse.quote(data) != data
+        ]
+        if self.operation_logger:
+            self.operation_logger.data_to_redact.extend(data_to_redact)
+        elif data_to_redact:
+            raise YunohostError("app_argument_cant_redact", arg=question.name)
+
         return question.value
 
 
@@ -2954,12 +2944,6 @@ class StringArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "string"
     default_value = ""
 
-    def _prevalidate(self, question):
-        super()._prevalidate(question)
-        raise YunohostValidationError(
-            "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2")
-        )
-
 class TagsArgumentParser(YunoHostArgumentFormatParser):
     argument_type = "tags"
 
@@ -2982,7 +2966,7 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser):
         question = super(PasswordArgumentParser, self).parse_question(
             question, user_answers
         )
-
+        question.redact = True
         if question.default is not None:
             raise YunohostValidationError(
                 "app_argument_password_no_default", name=question.name
@@ -3242,6 +3226,8 @@ class FileArgumentParser(YunoHostArgumentFormatParser):
             # os.path.join to avoid the user to be able to rewrite a file in filesystem
             # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
             file_path = os.path.normpath(upload_dir + "/" + filename)
+            if not file_path.startswith(upload_dir + "/"):
+                raise YunohostError("relative_parent_path_in_filename_forbidden")
             i = 2
             while os.path.exists(file_path):
                 file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))

From 60564d55c8899d51dbff7cec32e5b7107c045959 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Thu, 2 Sep 2021 17:14:27 +0200
Subject: [PATCH 46/51] (enh] Config panel helpers wording

---
 data/helpers.d/configpanel | 139 ++++++-------------------------------
 data/helpers.d/utils       |  98 ++++++++++++++++++++++++++
 2 files changed, 119 insertions(+), 118 deletions(-)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/configpanel
index b55b0b0fd..7727c8d55 100644
--- a/data/helpers.d/configpanel
+++ b/data/helpers.d/configpanel
@@ -1,105 +1,7 @@
 #!/bin/bash
 
 
-# Get a value from heterogeneous file (yaml, json, php, python...)
-#
-# usage: ynh_value_get --file=PATH --key=KEY
-# | arg: -f, --file=     - the path to the file
-# | arg: -k, --key=     - the key to get
-#
-# This helpers match several var affectation use case in several languages
-# We don't use jq or equivalent to keep comments and blank space in files
-# This helpers work line by line, it is not able to work correctly 
-# if you have several identical keys in your files
-# 
-# Example of line this helpers can managed correctly
-# .yml
-#     title: YunoHost documentation
-#     email: 'yunohost@yunohost.org'
-# .json
-#     "theme": "colib'ris",
-#     "port": 8102
-#     "some_boolean":     false,
-#     "user": null
-# .ini
-#     some_boolean = On
-#     action = "Clear"
-#     port = 20
-# .php
-#     $user=
-#     user => 20
-# .py
-#     USER = 8102
-#     user = 'https://donate.local'
-#     CUSTOM['user'] = 'YunoHost'
-# Requires YunoHost version 4.3 or higher.
-ynh_value_get() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=fk
-    local -A args_array=( [f]=file= [k]=key= )
-    local file
-    local key
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-
-    local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
-    
-    local crazy_value="$((grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL)  | head -n1)"
-    #"
-    
-    local first_char="${crazy_value:0:1}"
-    if [[ "$first_char" == '"' ]] ; then
-        echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1  | sed 's/\\"/"/g'
-    elif [[ "$first_char" == "'" ]] ; then
-        echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1  | sed "s/\\\\'/'/g"
-    else
-        echo "$crazy_value"
-    fi
-}
-
-# Set a value into heterogeneous file (yaml, json, php, python...)
-#
-# usage: ynh_value_set --file=PATH --key=KEY --value=VALUE
-# | arg: -f, --file=     - the path to the file
-# | arg: -k, --key=     - the key to set
-# | arg: -v, --value=     - the value to set
-#
-# Requires YunoHost version 4.3 or higher.
-ynh_value_set() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=fkv
-    local -A args_array=( [f]=file= [k]=key= [v]=value=)
-    local file
-    local key
-    local value
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-    local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
-    
-    local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)"
-    # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
-    local first_char="${crazy_value:0:1}"
-    if [[ "$first_char" == '"' ]] ; then
-        # \ and sed is quite complex you need 2 \\ to get one in a sed
-        # So we need \\\\ to go through 2 sed
-        value="$(echo "$value" | sed 's/"/\\\\"/g')"
-        sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file}
-    elif [[ "$first_char" == "'" ]] ; then
-        # \ and sed is quite complex you need 2 \\ to get one in a sed
-        # However double quotes implies to double \\ to
-        # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str
-        value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")"
-        sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file}
-    else
-        if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
-            value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"'
-        fi
-        sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file}
-    fi
-    ynh_print_info "Configuration key '$key' edited into $file"
-}
-
-_ynh_panel_get() {
+_ynh_app_config_get() {
     # From settings
     local lines
     lines=`python3 << EOL
@@ -165,7 +67,7 @@ EOL
             local source_key="$(echo "$source" | cut -d: -f1)"
             source_key=${source_key:-$short_setting}
             local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
-            old[$short_setting]="$(ynh_value_get --file="${source_file}" --key="${source_key}")"
+            old[$short_setting]="$(ynh_get_var --file="${source_file}" --key="${source_key}")"
         
         fi
     done
@@ -173,7 +75,7 @@ EOL
     
 }
 
-_ynh_panel_apply() {
+_ynh_app_config_apply() {
     for short_setting in "${!old[@]}"
     do
         local setter="set__${short_setting}"
@@ -222,18 +124,19 @@ _ynh_panel_apply() {
                 local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
                 
                 ynh_backup_if_checksum_is_different --file="$source_file"
-                ynh_value_set --file="${source_file}" --key="${source_key}" --value="${!short_setting}"
+                ynh_set_var --file="${source_file}" --key="${source_key}" --value="${!short_setting}"
                 ynh_store_file_checksum --file="$source_file"
                 
                 # We stored the info in settings in order to be able to upgrade the app
                 ynh_app_setting_set $app $short_setting "${!short_setting}"
+                ynh_print_info "Configuration key '$source_key' edited into $source_file"
 
             fi
         fi
     done
 }
 
-_ynh_panel_show() {
+_ynh_app_config_show() {
     for short_setting in "${!old[@]}"
     do
         if [[ "${old[$short_setting]}" != YNH_NULL ]] ; then
@@ -248,7 +151,7 @@ _ynh_panel_show() {
     done
 }
 
-_ynh_panel_validate() {
+_ynh_app_config_validate() {
     # Change detection
     ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1
     local is_error=true
@@ -319,23 +222,23 @@ _ynh_panel_validate() {
     
 }
 
-ynh_panel_get() {
-    _ynh_panel_get
+ynh_app_config_get() {
+    _ynh_app_config_get
 }
 
-ynh_panel_show() {
-    _ynh_panel_show
+ynh_app_config_show() {
+    _ynh_app_config_show
 }
 
-ynh_panel_validate() {
-    _ynh_panel_validate
+ynh_app_config_validate() {
+    _ynh_app_config_validate
 }
 
-ynh_panel_apply() {
-    _ynh_panel_apply
+ynh_app_config_apply() {
+    _ynh_app_config_apply
 }
 
-ynh_panel_run() {
+ynh_app_config_run() {
     declare -Ag old=()
     declare -Ag changed=()
     declare -Ag file_hash=()
@@ -345,18 +248,18 @@ ynh_panel_run() {
     
     case $1 in
         show)
-            ynh_panel_get
-            ynh_panel_show
+            ynh_app_config_get
+            ynh_app_config_show
             ;;
         apply) 
             max_progression=4
             ynh_script_progression --message="Reading config panel description and current configuration..."
-            ynh_panel_get
+            ynh_app_config_get
             
-            ynh_panel_validate
+            ynh_app_config_validate
             
             ynh_script_progression --message="Applying the new configuration..."
-            ynh_panel_apply
+            ynh_app_config_apply
             ynh_script_progression --message="Configuration of $app completed" --last
             ;;
     esac
diff --git a/data/helpers.d/utils b/data/helpers.d/utils
index 00bec89ac..1c4f73ddf 100644
--- a/data/helpers.d/utils
+++ b/data/helpers.d/utils
@@ -473,6 +473,104 @@ ynh_replace_vars () {
     done
 }
 
+# Get a value from heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_get_var --file=PATH --key=KEY
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to get
+#
+# This helpers match several var affectation use case in several languages
+# We don't use jq or equivalent to keep comments and blank space in files
+# This helpers work line by line, it is not able to work correctly 
+# if you have several identical keys in your files
+# 
+# Example of line this helpers can managed correctly
+# .yml
+#     title: YunoHost documentation
+#     email: 'yunohost@yunohost.org'
+# .json
+#     "theme": "colib'ris",
+#     "port": 8102
+#     "some_boolean":     false,
+#     "user": null
+# .ini
+#     some_boolean = On
+#     action = "Clear"
+#     port = 20
+# .php
+#     $user=
+#     user => 20
+# .py
+#     USER = 8102
+#     user = 'https://donate.local'
+#     CUSTOM['user'] = 'YunoHost'
+# Requires YunoHost version 4.3 or higher.
+ynh_get_var() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fk
+    local -A args_array=( [f]=file= [k]=key= )
+    local file
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local var_part='^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
+    
+    local crazy_value="$((grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} || echo YNH_NULL)  | head -n1)"
+    #"
+    
+    local first_char="${crazy_value:0:1}"
+    if [[ "$first_char" == '"' ]] ; then
+        echo "$crazy_value" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1  | sed 's/\\"/"/g'
+    elif [[ "$first_char" == "'" ]] ; then
+        echo "$crazy_value" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1  | sed "s/\\\\'/'/g"
+    else
+        echo "$crazy_value"
+    fi
+}
+
+# Set a value into heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_set_var --file=PATH --key=KEY --value=VALUE
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to set
+# | arg: -v, --value=     - the value to set
+#
+# Requires YunoHost version 4.3 or higher.
+ynh_set_var() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fkv
+    local -A args_array=( [f]=file= [k]=key= [v]=value=)
+    local file
+    local key
+    local value
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    local var_part='[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*'
+    
+    local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)"
+    # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
+    local first_char="${crazy_value:0:1}"
+    if [[ "$first_char" == '"' ]] ; then
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # So we need \\\\ to go through 2 sed
+        value="$(echo "$value" | sed 's/"/\\\\"/g')"
+        sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file}
+    elif [[ "$first_char" == "'" ]] ; then
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # However double quotes implies to double \\ to
+        # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str
+        value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")"
+        sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file}
+    else
+        if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
+            value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"'
+        fi
+        sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file}
+    fi
+}
+
+
 # Render templates with Jinja2
 #
 # [internal]

From ea2026a0297c75f8616373b5120cc7b28a5379f2 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Thu, 2 Sep 2021 17:15:38 +0200
Subject: [PATCH 47/51] (enh] Config panel helpers wording

---
 data/helpers.d/{configpanel => config} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename data/helpers.d/{configpanel => config} (100%)

diff --git a/data/helpers.d/configpanel b/data/helpers.d/config
similarity index 100%
rename from data/helpers.d/configpanel
rename to data/helpers.d/config

From edef077c7419782b90436098413a682bc65febb6 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Thu, 2 Sep 2021 17:19:30 +0200
Subject: [PATCH 48/51] (enh] Config panel helpers wording

---
 src/yunohost/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index a4c215328..828175393 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1945,7 +1945,7 @@ def _call_config_script(operation_logger, app, action, env={}, config_panel=None
 source /usr/share/yunohost/helpers
 ynh_abort_if_errors
 final_path=$(ynh_app_setting_get $app final_path)
-ynh_panel_run $1
+ynh_app_config_run $1
 """
         write_to_file(config_script, default_script)
 

From c5885000ecac21d6b3620def3f784c0617b9f0d1 Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Thu, 2 Sep 2021 19:46:33 +0200
Subject: [PATCH 49/51] [enh] Better upgrade management

---
 data/helpers.d/backup | 15 ++++++++++++++-
 data/helpers.d/config | 10 ++++++++--
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/data/helpers.d/backup b/data/helpers.d/backup
index ae746a37b..21ca2d7f0 100644
--- a/data/helpers.d/backup
+++ b/data/helpers.d/backup
@@ -326,12 +326,25 @@ ynh_bind_or_cp() {
 ynh_store_file_checksum () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    local -A args_array=( [f]=file= )
+    local -A args_array=( [f]=file= [u]=update_only )
     local file
+    local update_only
+    update_only="${update_only:-0}"
+
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
     local checksum_setting_name=checksum_${file//[\/ ]/_}    # Replace all '/' and ' ' by '_'
+    
+    # If update only, we don't save the new checksum if no old checksum exist
+    if [ $update_only -eq 1 ] ; then
+        local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
+        if [ -z "${checksum_value}" ] ; then
+            unset backup_file_checksum
+            return 0
+        fi
+    fi
+    
     ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
 
     # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
diff --git a/data/helpers.d/config b/data/helpers.d/config
index 7727c8d55..52454ff91 100644
--- a/data/helpers.d/config
+++ b/data/helpers.d/config
@@ -96,10 +96,14 @@ _ynh_app_config_apply() {
                 fi
                 local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
                 if [[ "${!short_setting}" == "" ]] ; then
+                    ynh_backup_if_checksum_is_different --file="$source_file"
                     rm -f "$source_file"
+                    ynh_delete_file_checksum --file="$source_file" --update_only
                     ynh_print_info "File '$source_file' removed"
                 else
+                    ynh_backup_if_checksum_is_different --file="$source_file"
                     cp "${!short_setting}" "$source_file"
+                    ynh_store_file_checksum --file="$source_file" --update_only
                     ynh_print_info "File '$source_file' overwrited with ${!short_setting}"
                 fi
             
@@ -114,7 +118,9 @@ _ynh_app_config_apply() {
                     ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
                 fi
                 local source_file="$(echo "$source" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+                ynh_backup_if_checksum_is_different --file="$source_file"
                 echo "${!short_setting}" > "$source_file"
+                ynh_store_file_checksum --file="$source_file" --update_only
                 ynh_print_info "File '$source_file' overwrited with the content you provieded in '${short_setting}' question"
 
             # Set value into a kind of key/value file
@@ -122,10 +128,10 @@ _ynh_app_config_apply() {
                 local source_key="$(echo "$source" | cut -d: -f1)"
                 source_key=${source_key:-$short_setting}
                 local source_file="$(echo "$source" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
-                
+
                 ynh_backup_if_checksum_is_different --file="$source_file"
                 ynh_set_var --file="${source_file}" --key="${source_key}" --value="${!short_setting}"
-                ynh_store_file_checksum --file="$source_file"
+                ynh_store_file_checksum --file="$source_file" --update_only
                 
                 # We stored the info in settings in order to be able to upgrade the app
                 ynh_app_setting_set $app $short_setting "${!short_setting}"

From e0fe82f566e15fc50376869e22a593aa91446fc4 Mon Sep 17 00:00:00 2001
From: "ljf (zamentur)" <zamentur@users.noreply.github.com>
Date: Thu, 2 Sep 2021 20:09:41 +0200
Subject: [PATCH 50/51] [fix] Some service has no test_conf

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>
---
 src/yunohost/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index 828175393..de6df6579 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -1908,7 +1908,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None, args_
         logger.debug(f"Reloading {service}")
         if not _run_service_command('reload-or-restart', service):
             services = _get_services()
-            test_conf = services[service].get('test_conf')
+            test_conf = services[service].get('test_conf', 'true')
             errors = check_output(f"{test_conf}; exit 0") if test_conf else ''
             raise YunohostError(
                 "app_config_failed_service_reload",

From b28cf8cbce8a9126a5a97af4f1fd58d21e73e8df Mon Sep 17 00:00:00 2001
From: ljf <ljf+git@grimaud.me>
Date: Fri, 3 Sep 2021 14:26:34 +0200
Subject: [PATCH 51/51] [enh] Prepare config panel for domain

---
 data/helpers.d/utils         |   7 +-
 src/yunohost/app.py          | 947 +++--------------------------------
 src/yunohost/utils/config.py | 814 ++++++++++++++++++++++++++++++
 src/yunohost/utils/i18n.py   |  46 ++
 4 files changed, 921 insertions(+), 893 deletions(-)
 create mode 100644 src/yunohost/utils/config.py
 create mode 100644 src/yunohost/utils/i18n.py

diff --git a/data/helpers.d/utils b/data/helpers.d/utils
index 1c4f73ddf..14e7ebe4a 100644
--- a/data/helpers.d/utils
+++ b/data/helpers.d/utils
@@ -551,22 +551,23 @@ ynh_set_var() {
     local crazy_value="$(grep -i -o -P '^[ \t]*\$?(\w*\[)?[ \t]*["'"']?${key}['"'"]?[ \t]*\]?[ \t]*[:=]>?[ \t]*\K.*(?=[ \t,\n;]*$)' ${file} | head -n1)"
     # local crazy_value="$(grep -i -o -P "^${var_part}\K.*(?=[ \t,\n;]*\$)" ${file} | head -n1)"
     local first_char="${crazy_value:0:1}"
+    delimiter=$'\001'
     if [[ "$first_char" == '"' ]] ; then
         # \ and sed is quite complex you need 2 \\ to get one in a sed
         # So we need \\\\ to go through 2 sed
         value="$(echo "$value" | sed 's/"/\\\\"/g')"
-        sed -ri 'sø^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$ø\1'"${value}"'\4øi' ${file}
+        sed -ri s$delimiter'^('"${var_part}"'")([^"]|\\")*("[ \t;,]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file}
     elif [[ "$first_char" == "'" ]] ; then
         # \ and sed is quite complex you need 2 \\ to get one in a sed
         # However double quotes implies to double \\ to
         # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str
         value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")"
-        sed -ri "sø^(${var_part}')([^']|\\')*('"'[ \t,;]*)$ø\1'"${value}"'\4øi' ${file}
+        sed -ri "s$delimiter^(${var_part}')([^']|\\')*('"'[ \t,;]*)$'$delimiter'\1'"${value}"'\4'$delimiter'i' ${file}
     else
         if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] ; then
             value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"'
         fi
-        sed -ri "sø^(${var_part}).*"'$ø\1'"${value}"'øi' ${file}
+        sed -ri "s$delimiter^(${var_part}).*"'$'$delimiter'\1'"${value}"$delimiter'i' ${file}
     fi
 }
 
diff --git a/src/yunohost/app.py b/src/yunohost/app.py
index de6df6579..522f695e2 100644
--- a/src/yunohost/app.py
+++ b/src/yunohost/app.py
@@ -33,7 +33,6 @@ import re
 import subprocess
 import glob
 import urllib.parse
-import base64
 import tempfile
 from collections import OrderedDict
 
@@ -54,8 +53,10 @@ from moulinette.utils.filesystem import (
     mkdir,
 )
 
-from yunohost.service import service_status, _run_service_command, _get_services
-from yunohost.utils import packages
+from yunohost.service import service_status, _run_service_command
+from yunohost.utils import packages, config
+from yunohost.utils.config import ConfigPanel, parse_args_in_yunohost_format, YunoHostArgumentFormatParser
+from yunohost.utils.i18n import _value_for_locale
 from yunohost.utils.error import YunohostError, YunohostValidationError
 from yunohost.utils.filesystem import free_space_in_directory
 from yunohost.log import is_unit_operation, OperationLogger
@@ -70,7 +71,6 @@ APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml"
 APPS_CATALOG_API_VERSION = 2
 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
 
-APPS_CONFIG_PANEL_VERSION_SUPPORTED = 1.0
 re_app_instance_name = re.compile(
     r"^(?P<appid>[\w-]+?)(__(?P<appinstancenb>[1-9][0-9]*))?$"
 )
@@ -1756,51 +1756,12 @@ def app_action_run(operation_logger, app, action, args=None):
     return logger.success("Action successed!")
 
 
-@is_unit_operation()
-def app_config_get(operation_logger, app, key='', mode='classic'):
+def app_config_get(app, key='', mode='classic'):
     """
     Display an app configuration in classic, full or export mode
     """
-
-    # Check app is installed
-    _assert_is_installed(app)
-
-    filter_key = key or ''
-
-    # Read config panel toml
-    config_panel = _get_app_config_panel(app, filter_key=filter_key)
-
-    if not config_panel:
-        raise YunohostError("app_config_no_panel")
-
-    # Call config script in order to hydrate config panel with current values
-    values = _call_config_script(operation_logger, app, 'show', config_panel=config_panel)
-
-    # Format result in full mode
-    if mode == 'full':
-        operation_logger.success()
-        return config_panel
-
-    # In 'classic' mode, we display the current value if key refer to an option
-    if filter_key.count('.') == 2 and mode == 'classic':
-        option = filter_key.split('.')[-1]
-        operation_logger.success()
-        return values.get(option, None)
-
-    # Format result in 'classic' or 'export' mode
-    logger.debug(f"Formating result in '{mode}' mode")
-    result = {}
-    for panel, section, option in _get_config_iterator(config_panel):
-        key = f"{panel['id']}.{section['id']}.{option['id']}"
-        if mode == 'export':
-            result[option['id']] = option.get('current_value')
-        else:
-            result[key] = { 'ask': _value_for_locale(option['ask']) }
-            if 'current_value' in option:
-                result[key]['value'] = option['current_value']
-
-    operation_logger.success()
-    return result
+    config = AppConfigPanel(app)
+    return config.get(key, mode)
 
 
 @is_unit_operation()
@@ -1809,182 +1770,65 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None, args_
     Apply a new app configuration
     """
 
-    # Check app is installed
-    _assert_is_installed(app)
-
-    filter_key = key or ''
-
-    # Read config panel toml
-    config_panel = _get_app_config_panel(app, filter_key=filter_key)
-
-    if not config_panel:
-        raise YunohostError("app_config_no_panel")
-
-    if (args is not None or args_file is not None) and value is not None:
-        raise YunohostError("app_config_args_value")
-
-    if filter_key.count('.') != 2 and not value is None:
-        raise YunohostError("app_config_set_value_on_section")
-
-    # Import and parse pre-answered options
-    logger.debug("Import and parse pre-answered options")
-    args = urllib.parse.parse_qs(args or '', keep_blank_values=True)
-    args = { key: ','.join(value_) for key, value_ in args.items() }
-
-    if args_file:
-        # Import YAML / JSON file but keep --args values
-        args = { **read_yaml(args_file), **args }
-
-    if value is not None:
-        args = {filter_key.split('.')[-1]: value}
-
-    # Call config script in order to hydrate config panel with current values
-    _call_config_script(operation_logger, app, 'show', config_panel=config_panel)
-
-    # Ask unanswered question and prevalidate
-    logger.debug("Ask unanswered question and prevalidate data")
-    def display_header(message):
-        """ CLI panel/section header display
-        """
-        if Moulinette.interface.type == 'cli' and filter_key.count('.') < 2:
-            Moulinette.display(colorize(message, 'purple'))
-
-    try:
-        env = {}
-        for panel, section, obj in _get_config_iterator(config_panel,
-                                                    ['panel', 'section']):
-            if panel == obj:
-                name = _value_for_locale(panel['name'])
-                display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
-                continue
-            name = _value_for_locale(section['name'])
-            display_header(f"\n# {name}")
-
-            # Check and ask unanswered questions
-            env.update(_parse_args_in_yunohost_format(
-                args, section['options']
-            ))
-
-        # Call config script in 'apply' mode
-        logger.info("Running config script...")
-        env = {key: str(value[0]) for key, value in env.items() if not value[0] is None}
-
-        errors = _call_config_script(operation_logger, app, 'apply', env=env)
-    # Script got manually interrupted ...
-    # N.B. : KeyboardInterrupt does not inherit from Exception
-    except (KeyboardInterrupt, EOFError):
-        error = m18n.n("operation_interrupted")
-        logger.error(m18n.n("app_config_failed", app=app, error=error))
-        failure_message_with_debug_instructions = operation_logger.error(error)
-        raise
-    # Something wrong happened in Yunohost's code (most probably hook_exec)
-    except Exception:
-        import traceback
-
-        error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
-        logger.error(m18n.n("app_config_failed", app=app, error=error))
-        failure_message_with_debug_instructions = operation_logger.error(error)
-        raise
-    finally:
-        # Delete files uploaded from API
-        FileArgumentParser.clean_upload_dirs()
-
-    if errors:
-        return {
-            "errors": errors,
-        }
-
-    # Reload services
-    logger.info("Reloading services...")
-    services_to_reload = set()
-    for panel, section, obj in _get_config_iterator(config_panel,
-                                                    ['panel', 'section', 'option']):
-        services_to_reload |= set(obj.get('services', []))
-
-    services_to_reload = list(services_to_reload)
-    services_to_reload.sort(key = 'nginx'.__eq__)
-    for service in services_to_reload:
-        service = service.replace('__APP__', app)
-        logger.debug(f"Reloading {service}")
-        if not _run_service_command('reload-or-restart', service):
-            services = _get_services()
-            test_conf = services[service].get('test_conf', 'true')
-            errors = check_output(f"{test_conf}; exit 0") if test_conf else ''
-            raise YunohostError(
-                "app_config_failed_service_reload",
-                service=service, errors=errors
-            )
-
-    logger.success("Config updated as expected")
-    return {}
-
-
-def _get_config_iterator(config_panel, trigger=['option']):
-    for panel in config_panel.get("panels", []):
-        if 'panel' in trigger:
-            yield (panel, None, panel)
-        for section in panel.get("sections", []):
-            if 'section' in trigger:
-                yield (panel, section, section)
-            if 'option' in trigger:
-                for option in section.get("options", []):
-                    yield (panel, section, option)
-
-
-def _call_config_script(operation_logger, app, action, env={}, config_panel=None):
-    from yunohost.hook import hook_exec
+    config = AppConfigPanel(app)
 
     YunoHostArgumentFormatParser.operation_logger = operation_logger
     operation_logger.start()
 
-    # Add default config script if needed
-    config_script = os.path.join(APPS_SETTING_PATH, app, "scripts", "config")
-    if not os.path.exists(config_script):
-        logger.debug("Adding a default config script")
-        default_script = """#!/bin/bash
+    result = config.set(key, value, args, args_file)
+    if "errors" not in result:
+        operation_logger.success()
+    return result
+
+class AppConfigPanel(ConfigPanel):
+    def __init__(self, app):
+
+        # Check app is installed
+        _assert_is_installed(app)
+
+        self.app = app
+        config_path = os.path.join(APPS_SETTING_PATH, app, "config_panel.toml")
+        super().__init__(config_path=config_path)
+
+    def _load_current_values(self):
+        self.values = self._call_config_script('show')
+
+    def _apply(self):
+        self.errors = self._call_config_script('apply', self.new_values)
+
+    def _call_config_script(self, action, env={}):
+        from yunohost.hook import hook_exec
+
+        # Add default config script if needed
+        config_script = os.path.join(APPS_SETTING_PATH, self.app, "scripts", "config")
+        if not os.path.exists(config_script):
+            logger.debug("Adding a default config script")
+            default_script = """#!/bin/bash
 source /usr/share/yunohost/helpers
 ynh_abort_if_errors
 final_path=$(ynh_app_setting_get $app final_path)
 ynh_app_config_run $1
 """
-        write_to_file(config_script, default_script)
+            write_to_file(config_script, default_script)
 
-    # Call config script to extract current values
-    logger.debug(f"Calling '{action}' action from config script")
-    app_id, app_instance_nb = _parse_app_instance_name(app)
-    env.update({
-        "app_id": app_id,
-        "app": app,
-        "app_instance_nb": str(app_instance_nb),
-    })
-
-    ret, parsed_values = hook_exec(
-        config_script, args=[action], env=env
-    )
-    if ret != 0:
-        if action == 'show':
-            raise YunohostError("app_config_unable_to_read_values")
-        else:
-            raise YunohostError("app_config_unable_to_apply_values_correctly")
-
-        return parsed_values
-
-    if not config_panel:
-        return parsed_values
-
-    # Hydrating config panel with current value
-    logger.debug("Hydrating config with current values")
-    for _, _, option in _get_config_iterator(config_panel):
-        if option['name'] not in parsed_values:
-            continue
-        value = parsed_values[option['name']]
-        # In general, the value is just a simple value.
-        # Sometimes it could be a dict used to overwrite the option itself
-        value = value if isinstance(value, dict) else {'current_value': value }
-        option.update(value)
-
-    return parsed_values
+        # Call config script to extract current values
+        logger.debug(f"Calling '{action}' action from config script")
+        app_id, app_instance_nb = _parse_app_instance_name(self.app)
+        env.update({
+            "app_id": app_id,
+            "app": self.app,
+            "app_instance_nb": str(app_instance_nb),
+        })
 
+        ret, values = hook_exec(
+            config_script, args=[action], env=env
+        )
+        if ret != 0:
+            if action == 'show':
+                raise YunohostError("app_config_unable_to_read_values")
+            else:
+                raise YunohostError("app_config_unable_to_apply_values_correctly")
+        return values
 
 def _get_all_installed_apps_id():
     """
@@ -2087,163 +1931,6 @@ def _get_app_actions(app_id):
     return None
 
 
-def _get_app_config_panel(app_id, filter_key=''):
-    "Get app config panel stored in json or in toml"
-
-    # Split filter_key
-    filter_key = dict(enumerate(filter_key.split('.')))
-    if len(filter_key) > 3:
-        raise YunohostError("app_config_too_much_sub_keys")
-
-    # Open TOML
-    config_panel_toml_path = os.path.join(
-        APPS_SETTING_PATH, app_id, "config_panel.toml"
-    )
-
-    # sample data to get an idea of what is going on
-    # this toml extract:
-    #
-    # version = "0.1"
-    # name = "Unattended-upgrades configuration panel"
-    #
-    # [main]
-    # name = "Unattended-upgrades configuration"
-    #
-    #     [main.unattended_configuration]
-    #     name = "50unattended-upgrades configuration file"
-    #
-    #         [main.unattended_configuration.upgrade_level]
-    #         name = "Choose the sources of packages to automatically upgrade."
-    #         default = "Security only"
-    #         type = "text"
-    #         help = "We can't use a choices field for now. In the meantime[...]"
-    #         # choices = ["Security only", "Security and updates"]
-
-    #         [main.unattended_configuration.ynh_update]
-    #         name = "Would you like to update YunoHost packages automatically ?"
-    #         type = "bool"
-    #         default = true
-    #
-    # will be parsed into this:
-    #
-    # OrderedDict([(u'version', u'0.1'),
-    #              (u'name', u'Unattended-upgrades configuration panel'),
-    #              (u'main',
-    #               OrderedDict([(u'name', u'Unattended-upgrades configuration'),
-    #                            (u'unattended_configuration',
-    #                             OrderedDict([(u'name',
-    #                                           u'50unattended-upgrades configuration file'),
-    #                                          (u'upgrade_level',
-    #                                           OrderedDict([(u'name',
-    #                                                         u'Choose the sources of packages to automatically upgrade.'),
-    #                                                        (u'default',
-    #                                                         u'Security only'),
-    #                                                        (u'type', u'text'),
-    #                                                        (u'help',
-    #                                                         u"We can't use a choices field for now. In the meantime please choose between one of this values:<br>Security only, Security and updates.")])),
-    #                                          (u'ynh_update',
-    #                                           OrderedDict([(u'name',
-    #                                                         u'Would you like to update YunoHost packages automatically ?'),
-    #                                                        (u'type', u'bool'),
-    #                                                        (u'default', True)])),
-    #
-    # and needs to be converted into this:
-    #
-    # {u'name': u'Unattended-upgrades configuration panel',
-    #  u'panel': [{u'id': u'main',
-    #    u'name': u'Unattended-upgrades configuration',
-    #    u'sections': [{u'id': u'unattended_configuration',
-    #      u'name': u'50unattended-upgrades configuration file',
-    #      u'options': [{u'//': u'"choices" : ["Security only", "Security and updates"]',
-    #        u'default': u'Security only',
-    #        u'help': u"We can't use a choices field for now. In the meantime[...]",
-    #        u'id': u'upgrade_level',
-    #        u'name': u'Choose the sources of packages to automatically upgrade.',
-    #        u'type': u'text'},
-    #       {u'default': True,
-    #        u'id': u'ynh_update',
-    #        u'name': u'Would you like to update YunoHost packages automatically ?',
-    #        u'type': u'bool'},
-
-    if not os.path.exists(config_panel_toml_path):
-        return None
-    toml_config_panel = read_toml(config_panel_toml_path)
-
-    # Check TOML config panel is in a supported version
-    if float(toml_config_panel["version"]) < APPS_CONFIG_PANEL_VERSION_SUPPORTED:
-        raise YunohostError(
-            "app_config_too_old_version", app=app_id,
-            version=toml_config_panel["version"]
-        )
-
-    # Transform toml format into internal format
-    defaults = {
-        'toml': {
-            'version': 1.0
-        },
-        'panels': {
-            'name': '',
-            'services': [],
-            'actions': {'apply': {'en': 'Apply'}}
-        }, # help
-        'sections': {
-            'name': '',
-            'services': [],
-            'optional': True
-        }, # visibleIf help
-        'options': {}
-        # ask type source help helpLink example style icon placeholder visibleIf
-        # optional choices pattern limit min max step accept redact
-    }
-
-    def convert(toml_node, node_type):
-        """Convert TOML in internal format ('full' mode used by webadmin)
-
-        Here are some properties of 1.0 config panel in toml:
-          - node properties and node children are mixed,
-          - text are in english only
-          - some properties have default values
-        This function detects all children nodes and put them in a list
-        """
-        # Prefill the node default keys if needed
-        default = defaults[node_type]
-        node = {key: toml_node.get(key, value) for key, value in default.items()}
-
-        # Define the filter_key part to use and the children type
-        i = list(defaults).index(node_type)
-        search_key = filter_key.get(i)
-        subnode_type = list(defaults)[i+1] if node_type != 'options' else None
-
-        for key, value in toml_node.items():
-            # Key/value are a child node
-            if isinstance(value, OrderedDict) and key not in default and subnode_type:
-                # We exclude all nodes not referenced by the filter_key
-                if search_key and key != search_key:
-                    continue
-                subnode = convert(value, subnode_type)
-                subnode['id'] = key
-                if node_type == 'sections':
-                    subnode['name'] = key # legacy
-                    subnode.setdefault('optional', toml_node.get('optional', True))
-                node.setdefault(subnode_type, []).append(subnode)
-            # Key/value are a property
-            else:
-                # Todo search all i18n keys
-                node[key] = value if key not in ['ask', 'help', 'name'] else { 'en': value }
-        return node
-
-    config_panel = convert(toml_config_panel, 'toml')
-
-    try:
-        config_panel['panels'][0]['sections'][0]['options'][0]
-    except (KeyError, IndexError):
-        raise YunohostError(
-            "app_config_empty_or_bad_filter_key", app=app_id, filter_key=filter_key
-        )
-
-    return config_panel
-
-
 def _get_app_settings(app_id):
     """
     Get settings of an installed app
@@ -2695,30 +2382,6 @@ def _installed_apps():
     return os.listdir(APPS_SETTING_PATH)
 
 
-def _value_for_locale(values):
-    """
-    Return proper value for current locale
-
-    Keyword arguments:
-        values -- A dict of values associated to their locale
-
-    Returns:
-        An utf-8 encoded string
-
-    """
-    if not isinstance(values, dict):
-        return values
-
-    for lang in [m18n.locale, m18n.default_locale]:
-        try:
-            return values[lang]
-        except KeyError:
-            continue
-
-    # Fallback to first value
-    return list(values.values())[0]
-
-
 def _check_manifest_requirements(manifest, app_instance_name):
     """Check if required packages are met from the manifest"""
 
@@ -2765,7 +2428,7 @@ def _parse_args_from_manifest(manifest, action, args={}):
         return OrderedDict()
 
     action_args = manifest["arguments"][action]
-    return _parse_args_in_yunohost_format(args, action_args)
+    return parse_args_in_yunohost_format(args, action_args)
 
 
 def _parse_args_for_action(action, args={}):
@@ -2789,507 +2452,11 @@ def _parse_args_for_action(action, args={}):
 
     action_args = action["arguments"]
 
-    return _parse_args_in_yunohost_format(args, action_args)
+    return parse_args_in_yunohost_format(args, action_args)
 
 
-class Question:
-    "empty class to store questions information"
 
 
-class YunoHostArgumentFormatParser(object):
-    hide_user_input_in_prompt = False
-    operation_logger = None
-
-    def parse_question(self, question, user_answers):
-        parsed_question = Question()
-
-        parsed_question.name = question["name"]
-        parsed_question.type = question.get("type", 'string')
-        parsed_question.default = question.get("default", None)
-        parsed_question.current_value = question.get("current_value")
-        parsed_question.optional = question.get("optional", False)
-        parsed_question.choices = question.get("choices", [])
-        parsed_question.pattern = question.get("pattern")
-        parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"})
-        parsed_question.help = question.get("help")
-        parsed_question.helpLink = question.get("helpLink")
-        parsed_question.value = user_answers.get(parsed_question.name)
-        parsed_question.redact = question.get('redact', False)
-
-        # Empty value is parsed as empty string
-        if parsed_question.default == "":
-            parsed_question.default = None
-
-        return parsed_question
-
-    def parse(self, question, user_answers):
-        question = self.parse_question(question, user_answers)
-
-        while True:
-            # Display question if no value filled or if it's a readonly message
-            if Moulinette.interface.type== 'cli':
-                text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
-                    question
-                )
-                if getattr(self, "readonly", False):
-                    Moulinette.display(text_for_user_input_in_cli)
-
-                elif question.value is None:
-                    prefill = ""
-                    if question.current_value is not None:
-                        prefill = question.current_value
-                    elif question.default is not None:
-                        prefill = question.default
-                    question.value = Moulinette.prompt(
-                        message=text_for_user_input_in_cli,
-                        is_password=self.hide_user_input_in_prompt,
-                        confirm=self.hide_user_input_in_prompt,
-                        prefill=prefill,
-                        is_multiline=(question.type == "text")
-                    )
-
-
-            # Apply default value
-            if question.value in [None, ""] and question.default is not None:
-                question.value = (
-                    getattr(self, "default_value", None)
-                    if question.default is None
-                    else question.default
-                )
-
-            # Prevalidation
-            try:
-                self._prevalidate(question)
-            except YunohostValidationError as e:
-                if Moulinette.interface.type== 'api':
-                    raise
-                Moulinette.display(str(e), 'error')
-                question.value = None
-                continue
-            break
-        # this is done to enforce a certain formating like for boolean
-        # by default it doesn't do anything
-        question.value = self._post_parse_value(question)
-
-        return (question.value, self.argument_type)
-
-    def _prevalidate(self, question):
-        if question.value in [None, ""] and not question.optional:
-            raise YunohostValidationError(
-                "app_argument_required", name=question.name
-            )
-
-        # we have an answer, do some post checks
-        if question.value is not None:
-            if question.choices and question.value not in question.choices:
-                self._raise_invalid_answer(question)
-            if question.pattern and not re.match(question.pattern['regexp'], str(question.value)):
-                raise YunohostValidationError(
-                    question.pattern['error'],
-                    name=question.name,
-                    value=question.value,
-                )
-
-    def _raise_invalid_answer(self, question):
-        raise YunohostValidationError(
-            "app_argument_choice_invalid",
-            name=question.name,
-            value=question.value,
-            choices=", ".join(question.choices),
-        )
-
-    def _format_text_for_user_input_in_cli(self, question):
-        text_for_user_input_in_cli = _value_for_locale(question.ask)
-
-        if question.choices:
-            text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices))
-
-        if question.help or question.helpLink:
-            text_for_user_input_in_cli += ":\033[m"
-        if question.help:
-            text_for_user_input_in_cli += "\n - "
-            text_for_user_input_in_cli += _value_for_locale(question.help)
-        if question.helpLink:
-            if not isinstance(question.helpLink, dict):
-                question.helpLink = {'href': question.helpLink}
-            text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}"
-        return text_for_user_input_in_cli
-
-    def _post_parse_value(self, question):
-        if not question.redact:
-            return question.value
-
-        # Tell the operation_logger to redact all password-type / secret args
-        # Also redact the % escaped version of the password that might appear in
-        # the 'args' section of metadata (relevant for password with non-alphanumeric char)
-        data_to_redact = []
-        if question.value and isinstance(question.value, str):
-            data_to_redact.append(question.value)
-        if question.current_value and isinstance(question.current_value, str):
-            data_to_redact.append(question.current_value)
-        data_to_redact += [
-            urllib.parse.quote(data)
-            for data in data_to_redact
-            if urllib.parse.quote(data) != data
-        ]
-        if self.operation_logger:
-            self.operation_logger.data_to_redact.extend(data_to_redact)
-        elif data_to_redact:
-            raise YunohostError("app_argument_cant_redact", arg=question.name)
-
-        return question.value
-
-
-class StringArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "string"
-    default_value = ""
-
-class TagsArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "tags"
-
-    def _prevalidate(self, question):
-        values = question.value
-        for value in values.split(','):
-            question.value = value
-            super()._prevalidate(question)
-        question.value = values
-
-
-
-class PasswordArgumentParser(YunoHostArgumentFormatParser):
-    hide_user_input_in_prompt = True
-    argument_type = "password"
-    default_value = ""
-    forbidden_chars = "{}"
-
-    def parse_question(self, question, user_answers):
-        question = super(PasswordArgumentParser, self).parse_question(
-            question, user_answers
-        )
-        question.redact = True
-        if question.default is not None:
-            raise YunohostValidationError(
-                "app_argument_password_no_default", name=question.name
-            )
-
-        return question
-
-    def _prevalidate(self, question):
-        super()._prevalidate(question)
-
-        if question.value is not None:
-            if any(char in question.value for char in self.forbidden_chars):
-                raise YunohostValidationError(
-                    "pattern_password_app", forbidden_chars=self.forbidden_chars
-                )
-
-            # If it's an optional argument the value should be empty or strong enough
-            from yunohost.utils.password import assert_password_is_strong_enough
-
-            assert_password_is_strong_enough("user", question.value)
-
-
-class PathArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "path"
-    default_value = ""
-
-
-class BooleanArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "boolean"
-    default_value = False
-
-    def parse_question(self, question, user_answers):
-        question = super().parse_question(
-            question, user_answers
-        )
-
-        if question.default is None:
-            question.default = False
-
-        return question
-
-    def _format_text_for_user_input_in_cli(self, question):
-        text_for_user_input_in_cli = _value_for_locale(question.ask)
-
-        text_for_user_input_in_cli += " [yes | no]"
-
-        if question.default is not None:
-            formatted_default = "yes" if question.default else "no"
-            text_for_user_input_in_cli += " (default: {0})".format(formatted_default)
-
-        return text_for_user_input_in_cli
-
-    def _post_parse_value(self, question):
-        if isinstance(question.value, bool):
-            return 1 if question.value else 0
-
-        if str(question.value).lower() in ["1", "yes", "y", "true"]:
-            return 1
-
-        if str(question.value).lower() in ["0", "no", "n", "false"]:
-            return 0
-
-        raise YunohostValidationError(
-            "app_argument_choice_invalid",
-            name=question.name,
-            value=question.value,
-            choices="yes, no, y, n, 1, 0",
-        )
-
-
-class DomainArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "domain"
-
-    def parse_question(self, question, user_answers):
-        from yunohost.domain import domain_list, _get_maindomain
-
-        question = super(DomainArgumentParser, self).parse_question(
-            question, user_answers
-        )
-
-        if question.default is None:
-            question.default = _get_maindomain()
-
-        question.choices = domain_list()["domains"]
-
-        return question
-
-    def _raise_invalid_answer(self, question):
-        raise YunohostValidationError(
-            "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown")
-        )
-
-
-class UserArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "user"
-
-    def parse_question(self, question, user_answers):
-        from yunohost.user import user_list, user_info
-        from yunohost.domain import _get_maindomain
-
-        question = super(UserArgumentParser, self).parse_question(
-            question, user_answers
-        )
-        question.choices = user_list()["users"]
-        if question.default is None:
-            root_mail = "root@%s" % _get_maindomain()
-            for user in question.choices.keys():
-                if root_mail in user_info(user).get("mail-aliases", []):
-                    question.default = user
-                    break
-
-        return question
-
-    def _raise_invalid_answer(self, question):
-        raise YunohostValidationError(
-            "app_argument_invalid",
-            field=question.name,
-            error=m18n.n("user_unknown", user=question.value),
-        )
-
-
-class NumberArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "number"
-    default_value = ""
-
-    def parse_question(self, question, user_answers):
-        question_parsed = super().parse_question(
-            question, user_answers
-        )
-        question_parsed.min = question.get('min', None)
-        question_parsed.max = question.get('max', None)
-        if question_parsed.default is None:
-            question_parsed.default = 0
-
-        return question_parsed
-
-    def _prevalidate(self, question):
-        super()._prevalidate(question)
-        if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()):
-            raise YunohostValidationError(
-                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
-            )
-
-        if question.min is not None and int(question.value) < question.min:
-            raise YunohostValidationError(
-                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
-            )
-
-        if question.max is not None and int(question.value) > question.max:
-            raise YunohostValidationError(
-                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
-            )
-
-    def _post_parse_value(self, question):
-        if isinstance(question.value, int):
-            return super()._post_parse_value(question)
-
-        if isinstance(question.value, str) and question.value.isdigit():
-            return int(question.value)
-
-        raise YunohostValidationError(
-            "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
-        )
-
-
-class DisplayTextArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "display_text"
-    readonly = True
-
-    def parse_question(self, question, user_answers):
-        question_parsed = super().parse_question(
-            question, user_answers
-        )
-
-        question_parsed.optional = True
-        question_parsed.style = question.get('style', 'info')
-
-        return question_parsed
-
-    def _format_text_for_user_input_in_cli(self, question):
-        text = question.ask['en']
-
-        if question.style in ['success', 'info', 'warning', 'danger']:
-            color = {
-                'success': 'green',
-                'info': 'cyan',
-                'warning': 'yellow',
-                'danger': 'red'
-            }
-            return colorize(m18n.g(question.style), color[question.style]) + f" {text}"
-        else:
-            return text
-
-class FileArgumentParser(YunoHostArgumentFormatParser):
-    argument_type = "file"
-    upload_dirs = []
-
-    @classmethod
-    def clean_upload_dirs(cls):
-        # Delete files uploaded from API
-        if Moulinette.interface.type== 'api':
-            for upload_dir in cls.upload_dirs:
-                if os.path.exists(upload_dir):
-                    shutil.rmtree(upload_dir)
-
-    def parse_question(self, question, user_answers):
-        question_parsed = super().parse_question(
-            question, user_answers
-        )
-        if question.get('accept'):
-            question_parsed.accept = question.get('accept').replace(' ', '').split(',')
-        else:
-            question_parsed.accept = []
-        if Moulinette.interface.type== 'api':
-            if user_answers.get(f"{question_parsed.name}[name]"):
-                question_parsed.value = {
-                    'content': question_parsed.value,
-                    'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name),
-                }
-        # If path file are the same
-        if question_parsed.value and str(question_parsed.value) == question_parsed.current_value:
-            question_parsed.value = None
-
-        return question_parsed
-
-    def _prevalidate(self, question):
-        super()._prevalidate(question)
-        if isinstance(question.value, str) and question.value and not os.path.exists(question.value):
-            raise YunohostValidationError(
-                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1")
-            )
-        if question.value in [None, ''] or not question.accept:
-            return
-
-        filename = question.value if isinstance(question.value, str) else question.value['filename']
-        if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept:
-            raise YunohostValidationError(
-                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2")
-            )
-
-
-    def _post_parse_value(self, question):
-        from base64 import b64decode
-        # Upload files from API
-        # A file arg contains a string with "FILENAME:BASE64_CONTENT"
-        if not question.value:
-            return question.value
-
-        if Moulinette.interface.type== 'api':
-
-            upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
-            FileArgumentParser.upload_dirs += [upload_dir]
-            filename = question.value['filename']
-            logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}")
-
-            # Filename is given by user of the API. For security reason, we have replaced
-            # os.path.join to avoid the user to be able to rewrite a file in filesystem
-            # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
-            file_path = os.path.normpath(upload_dir + "/" + filename)
-            if not file_path.startswith(upload_dir + "/"):
-                raise YunohostError("relative_parent_path_in_filename_forbidden")
-            i = 2
-            while os.path.exists(file_path):
-                file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
-                i += 1
-            content = question.value['content']
-            try:
-                with open(file_path, 'wb') as f:
-                    f.write(b64decode(content))
-            except IOError as e:
-                raise YunohostError("cannot_write_file", file=file_path, error=str(e))
-            except Exception as e:
-                raise YunohostError("error_writing_file", file=file_path, error=str(e))
-            question.value = file_path
-        return question.value
-
-
-ARGUMENTS_TYPE_PARSERS = {
-    "string": StringArgumentParser,
-    "text": StringArgumentParser,
-    "select": StringArgumentParser,
-    "tags": TagsArgumentParser,
-    "email": StringArgumentParser,
-    "url": StringArgumentParser,
-    "date": StringArgumentParser,
-    "time": StringArgumentParser,
-    "color": StringArgumentParser,
-    "password": PasswordArgumentParser,
-    "path": PathArgumentParser,
-    "boolean": BooleanArgumentParser,
-    "domain": DomainArgumentParser,
-    "user": UserArgumentParser,
-    "number": NumberArgumentParser,
-    "range": NumberArgumentParser,
-    "display_text": DisplayTextArgumentParser,
-    "alert": DisplayTextArgumentParser,
-    "markdown": DisplayTextArgumentParser,
-    "file": FileArgumentParser,
-}
-
-
-def _parse_args_in_yunohost_format(user_answers, argument_questions):
-    """Parse arguments store in either manifest.json or actions.json or from a
-    config panel against the user answers when they are present.
-
-    Keyword arguments:
-        user_answers -- a dictionnary of arguments from the user (generally
-                        empty in CLI, filed from the admin interface)
-        argument_questions -- the arguments description store in yunohost
-                              format from actions.json/toml, manifest.json/toml
-                              or config_panel.json/toml
-    """
-    parsed_answers_dict = OrderedDict()
-
-    for question in argument_questions:
-        parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]()
-
-        answer = parser.parse(question=question, user_answers=user_answers)
-        if answer is not None:
-            parsed_answers_dict[question["name"]] = answer
-
-    return parsed_answers_dict
-
 
 def _validate_and_normalize_webpath(args_dict, app_folder):
 
diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py
new file mode 100644
index 000000000..34883dcf7
--- /dev/null
+++ b/src/yunohost/utils/config.py
@@ -0,0 +1,814 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2018 YUNOHOST.ORG
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+import os
+import re
+import toml
+import urllib.parse
+import tempfile
+from collections import OrderedDict
+
+from moulinette.interfaces.cli import colorize
+from moulinette import Moulinette, m18n
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import (
+    read_toml,
+    read_yaml,
+    write_to_yaml,
+    mkdir,
+)
+
+from yunohost.service import _get_services
+from yunohost.service import _run_service_command, _get_services
+from yunohost.utils.i18n import _value_for_locale
+from yunohost.utils.error import YunohostError, YunohostValidationError
+
+logger = getActionLogger("yunohost.config")
+CONFIG_PANEL_VERSION_SUPPORTED = 1.0
+
+class ConfigPanel:
+
+    def __init__(self, config_path, save_path=None):
+        self.config_path = config_path
+        self.save_path = save_path
+        self.config = {}
+        self.values = {}
+        self.new_values = {}
+
+    def get(self, key='', mode='classic'):
+        self.filter_key = key or ''
+
+        # Read config panel toml
+        self._get_config_panel()
+
+        if not self.config:
+            raise YunohostError("config_no_panel")
+
+        # Read or get values and hydrate the config
+        self._load_current_values()
+        self._hydrate()
+
+        # Format result in full mode
+        if mode == 'full':
+            return self.config
+
+        # In 'classic' mode, we display the current value if key refer to an option
+        if self.filter_key.count('.') == 2 and mode == 'classic':
+            option = self.filter_key.split('.')[-1]
+            return self.values.get(option, None)
+
+        # Format result in 'classic' or 'export' mode
+        logger.debug(f"Formating result in '{mode}' mode")
+        result = {}
+        for panel, section, option in self._iterate():
+            key = f"{panel['id']}.{section['id']}.{option['id']}"
+            if mode == 'export':
+                result[option['id']] = option.get('current_value')
+            else:
+                result[key] = { 'ask': _value_for_locale(option['ask']) }
+                if 'current_value' in option:
+                    result[key]['value'] = option['current_value']
+
+        return result
+
+    def set(self, key=None, value=None, args=None, args_file=None):
+        self.filter_key = key or ''
+
+        # Read config panel toml
+        self._get_config_panel()
+
+        if not self.config:
+            raise YunohostError("config_no_panel")
+
+        if (args is not None or args_file is not None) and value is not None:
+            raise YunohostError("config_args_value")
+
+        if self.filter_key.count('.') != 2 and not value is None:
+            raise YunohostError("config_set_value_on_section")
+
+        # Import and parse pre-answered options
+        logger.debug("Import and parse pre-answered options")
+        args = urllib.parse.parse_qs(args or '', keep_blank_values=True)
+        self.args = { key: ','.join(value_) for key, value_ in args.items() }
+
+        if args_file:
+            # Import YAML / JSON file but keep --args values
+            self.args = { **read_yaml(args_file), **self.args }
+
+        if value is not None:
+            self.args = {self.filter_key.split('.')[-1]: value}
+
+        # Read or get values and hydrate the config
+        self._load_current_values()
+        self._hydrate()
+
+        try:
+            self._ask()
+            self._apply()
+
+        # Script got manually interrupted ...
+        # N.B. : KeyboardInterrupt does not inherit from Exception
+        except (KeyboardInterrupt, EOFError):
+            error = m18n.n("operation_interrupted")
+            logger.error(m18n.n("config_failed", error=error))
+            raise
+        # Something wrong happened in Yunohost's code (most probably hook_exec)
+        except Exception:
+            import traceback
+
+            error = m18n.n("unexpected_error", error="\n" + traceback.format_exc())
+            logger.error(m18n.n("config_failed", error=error))
+            raise
+        finally:
+            # Delete files uploaded from API
+            FileArgumentParser.clean_upload_dirs()
+
+        if self.errors:
+            return {
+                "errors": errors,
+            }
+
+        self._reload_services()
+
+        logger.success("Config updated as expected")
+        return {}
+
+    def _get_toml(self):
+        return read_toml(self.config_path)
+
+
+    def _get_config_panel(self):
+        # Split filter_key
+        filter_key = dict(enumerate(self.filter_key.split('.')))
+        if len(filter_key) > 3:
+            raise YunohostError("config_too_much_sub_keys")
+
+        if not os.path.exists(self.config_path):
+            return None
+        toml_config_panel = self._get_toml()
+
+        # Check TOML config panel is in a supported version
+        if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED:
+            raise YunohostError(
+                "config_too_old_version", version=toml_config_panel["version"]
+            )
+
+        # Transform toml format into internal format
+        defaults = {
+            'toml': {
+                'version': 1.0
+            },
+            'panels': {
+                'name': '',
+                'services': [],
+                'actions': {'apply': {'en': 'Apply'}}
+            }, # help
+            'sections': {
+                'name': '',
+                'services': [],
+                'optional': True
+            }, # visibleIf help
+            'options': {}
+            # ask type source help helpLink example style icon placeholder visibleIf
+            # optional choices pattern limit min max step accept redact
+        }
+
+        def convert(toml_node, node_type):
+            """Convert TOML in internal format ('full' mode used by webadmin)
+            Here are some properties of 1.0 config panel in toml:
+            - node properties and node children are mixed,
+            - text are in english only
+            - some properties have default values
+            This function detects all children nodes and put them in a list
+            """
+            # Prefill the node default keys if needed
+            default = defaults[node_type]
+            node = {key: toml_node.get(key, value) for key, value in default.items()}
+
+            # Define the filter_key part to use and the children type
+            i = list(defaults).index(node_type)
+            search_key = filter_key.get(i)
+            subnode_type = list(defaults)[i+1] if node_type != 'options' else None
+
+            for key, value in toml_node.items():
+                # Key/value are a child node
+                if isinstance(value, OrderedDict) and key not in default and subnode_type:
+                    # We exclude all nodes not referenced by the filter_key
+                    if search_key and key != search_key:
+                        continue
+                    subnode = convert(value, subnode_type)
+                    subnode['id'] = key
+                    if node_type == 'sections':
+                        subnode['name'] = key # legacy
+                        subnode.setdefault('optional', toml_node.get('optional', True))
+                    node.setdefault(subnode_type, []).append(subnode)
+                # Key/value are a property
+                else:
+                    # Todo search all i18n keys
+                    node[key] = value if key not in ['ask', 'help', 'name'] else { 'en': value }
+            return node
+
+        self.config = convert(toml_config_panel, 'toml')
+
+        try:
+            self.config['panels'][0]['sections'][0]['options'][0]
+        except (KeyError, IndexError):
+            raise YunohostError(
+                "config_empty_or_bad_filter_key", filter_key=self.filter_key
+            )
+
+        return self.config
+
+    def _hydrate(self):
+        # Hydrating config panel with current value
+        logger.debug("Hydrating config with current values")
+        for _, _, option in self._iterate():
+            if option['name'] not in self.values:
+                continue
+            value = self.values[option['name']]
+            # In general, the value is just a simple value.
+            # Sometimes it could be a dict used to overwrite the option itself
+            value = value if isinstance(value, dict) else {'current_value': value }
+            option.update(value)
+
+        return self.values
+
+    def _ask(self):
+        logger.debug("Ask unanswered question and prevalidate data")
+        def display_header(message):
+            """ CLI panel/section header display
+            """
+            if Moulinette.interface.type == 'cli' and self.filter_key.count('.') < 2:
+                Moulinette.display(colorize(message, 'purple'))
+        for panel, section, obj in self._iterate(['panel', 'section']):
+            if panel == obj:
+                name = _value_for_locale(panel['name'])
+                display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}")
+                continue
+            name = _value_for_locale(section['name'])
+            display_header(f"\n# {name}")
+
+            # Check and ask unanswered questions
+            self.new_values.update(parse_args_in_yunohost_format(
+                self.args, section['options']
+            ))
+        self.new_values = {key: str(value[0]) for key, value in self.new_values.items() if not value[0] is None}
+
+    def _apply(self):
+        logger.info("Running config script...")
+        dir_path = os.path.dirname(os.path.realpath(self.save_path))
+        if not os.path.exists(dir_path):
+            mkdir(dir_path, mode=0o700)
+        # Save the settings to the .yaml file
+        write_to_yaml(self.save_path, self.new_values)
+
+
+    def _reload_services(self):
+        logger.info("Reloading services...")
+        services_to_reload = set()
+        for panel, section, obj in self._iterate(['panel', 'section', 'option']):
+            services_to_reload |= set(obj.get('services', []))
+
+        services_to_reload = list(services_to_reload)
+        services_to_reload.sort(key = 'nginx'.__eq__)
+        for service in services_to_reload:
+            if '__APP__':
+                service = service.replace('__APP__', self.app)
+            logger.debug(f"Reloading {service}")
+            if not _run_service_command('reload-or-restart', service):
+                services = _get_services()
+                test_conf = services[service].get('test_conf', 'true')
+                errors = check_output(f"{test_conf}; exit 0") if test_conf else ''
+                raise YunohostError(
+                    "config_failed_service_reload",
+                    service=service, errors=errors
+                )
+
+    def _iterate(self, trigger=['option']):
+        for panel in self.config.get("panels", []):
+            if 'panel' in trigger:
+                yield (panel, None, panel)
+            for section in panel.get("sections", []):
+                if 'section' in trigger:
+                    yield (panel, section, section)
+                if 'option' in trigger:
+                    for option in section.get("options", []):
+                        yield (panel, section, option)
+
+
+class Question:
+    "empty class to store questions information"
+
+
+class YunoHostArgumentFormatParser(object):
+    hide_user_input_in_prompt = False
+    operation_logger = None
+
+    def parse_question(self, question, user_answers):
+        parsed_question = Question()
+
+        parsed_question.name = question["name"]
+        parsed_question.type = question.get("type", 'string')
+        parsed_question.default = question.get("default", None)
+        parsed_question.current_value = question.get("current_value")
+        parsed_question.optional = question.get("optional", False)
+        parsed_question.choices = question.get("choices", [])
+        parsed_question.pattern = question.get("pattern")
+        parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"})
+        parsed_question.help = question.get("help")
+        parsed_question.helpLink = question.get("helpLink")
+        parsed_question.value = user_answers.get(parsed_question.name)
+        parsed_question.redact = question.get('redact', False)
+
+        # Empty value is parsed as empty string
+        if parsed_question.default == "":
+            parsed_question.default = None
+
+        return parsed_question
+
+    def parse(self, question, user_answers):
+        question = self.parse_question(question, user_answers)
+
+        while True:
+            # Display question if no value filled or if it's a readonly message
+            if Moulinette.interface.type== 'cli':
+                text_for_user_input_in_cli = self._format_text_for_user_input_in_cli(
+                    question
+                )
+                if getattr(self, "readonly", False):
+                    Moulinette.display(text_for_user_input_in_cli)
+
+                elif question.value is None:
+                    prefill = ""
+                    if question.current_value is not None:
+                        prefill = question.current_value
+                    elif question.default is not None:
+                        prefill = question.default
+                    question.value = Moulinette.prompt(
+                        message=text_for_user_input_in_cli,
+                        is_password=self.hide_user_input_in_prompt,
+                        confirm=self.hide_user_input_in_prompt,
+                        prefill=prefill,
+                        is_multiline=(question.type == "text")
+                    )
+
+
+            # Apply default value
+            if question.value in [None, ""] and question.default is not None:
+                question.value = (
+                    getattr(self, "default_value", None)
+                    if question.default is None
+                    else question.default
+                )
+
+            # Prevalidation
+            try:
+                self._prevalidate(question)
+            except YunohostValidationError as e:
+                if Moulinette.interface.type== 'api':
+                    raise
+                Moulinette.display(str(e), 'error')
+                question.value = None
+                continue
+            break
+        # this is done to enforce a certain formating like for boolean
+        # by default it doesn't do anything
+        question.value = self._post_parse_value(question)
+
+        return (question.value, self.argument_type)
+
+    def _prevalidate(self, question):
+        if question.value in [None, ""] and not question.optional:
+            raise YunohostValidationError(
+                "app_argument_required", name=question.name
+            )
+
+        # we have an answer, do some post checks
+        if question.value is not None:
+            if question.choices and question.value not in question.choices:
+                self._raise_invalid_answer(question)
+            if question.pattern and not re.match(question.pattern['regexp'], str(question.value)):
+                raise YunohostValidationError(
+                    question.pattern['error'],
+                    name=question.name,
+                    value=question.value,
+                )
+
+    def _raise_invalid_answer(self, question):
+        raise YunohostValidationError(
+            "app_argument_choice_invalid",
+            name=question.name,
+            value=question.value,
+            choices=", ".join(question.choices),
+        )
+
+    def _format_text_for_user_input_in_cli(self, question):
+        text_for_user_input_in_cli = _value_for_locale(question.ask)
+
+        if question.choices:
+            text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices))
+
+        if question.help or question.helpLink:
+            text_for_user_input_in_cli += ":\033[m"
+        if question.help:
+            text_for_user_input_in_cli += "\n - "
+            text_for_user_input_in_cli += _value_for_locale(question.help)
+        if question.helpLink:
+            if not isinstance(question.helpLink, dict):
+                question.helpLink = {'href': question.helpLink}
+            text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}"
+        return text_for_user_input_in_cli
+
+    def _post_parse_value(self, question):
+        if not question.redact:
+            return question.value
+
+        # Tell the operation_logger to redact all password-type / secret args
+        # Also redact the % escaped version of the password that might appear in
+        # the 'args' section of metadata (relevant for password with non-alphanumeric char)
+        data_to_redact = []
+        if question.value and isinstance(question.value, str):
+            data_to_redact.append(question.value)
+        if question.current_value and isinstance(question.current_value, str):
+            data_to_redact.append(question.current_value)
+        data_to_redact += [
+            urllib.parse.quote(data)
+            for data in data_to_redact
+            if urllib.parse.quote(data) != data
+        ]
+        if self.operation_logger:
+            self.operation_logger.data_to_redact.extend(data_to_redact)
+        elif data_to_redact:
+            raise YunohostError("app_argument_cant_redact", arg=question.name)
+
+        return question.value
+
+
+class StringArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "string"
+    default_value = ""
+
+class TagsArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "tags"
+
+    def _prevalidate(self, question):
+        values = question.value
+        for value in values.split(','):
+            question.value = value
+            super()._prevalidate(question)
+        question.value = values
+
+
+
+class PasswordArgumentParser(YunoHostArgumentFormatParser):
+    hide_user_input_in_prompt = True
+    argument_type = "password"
+    default_value = ""
+    forbidden_chars = "{}"
+
+    def parse_question(self, question, user_answers):
+        question = super(PasswordArgumentParser, self).parse_question(
+            question, user_answers
+        )
+        question.redact = True
+        if question.default is not None:
+            raise YunohostValidationError(
+                "app_argument_password_no_default", name=question.name
+            )
+
+        return question
+
+    def _prevalidate(self, question):
+        super()._prevalidate(question)
+
+        if question.value is not None:
+            if any(char in question.value for char in self.forbidden_chars):
+                raise YunohostValidationError(
+                    "pattern_password_app", forbidden_chars=self.forbidden_chars
+                )
+
+            # If it's an optional argument the value should be empty or strong enough
+            from yunohost.utils.password import assert_password_is_strong_enough
+
+            assert_password_is_strong_enough("user", question.value)
+
+
+class PathArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "path"
+    default_value = ""
+
+
+class BooleanArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "boolean"
+    default_value = False
+
+    def parse_question(self, question, user_answers):
+        question = super().parse_question(
+            question, user_answers
+        )
+
+        if question.default is None:
+            question.default = False
+
+        return question
+
+    def _format_text_for_user_input_in_cli(self, question):
+        text_for_user_input_in_cli = _value_for_locale(question.ask)
+
+        text_for_user_input_in_cli += " [yes | no]"
+
+        if question.default is not None:
+            formatted_default = "yes" if question.default else "no"
+            text_for_user_input_in_cli += " (default: {0})".format(formatted_default)
+
+        return text_for_user_input_in_cli
+
+    def _post_parse_value(self, question):
+        if isinstance(question.value, bool):
+            return 1 if question.value else 0
+
+        if str(question.value).lower() in ["1", "yes", "y", "true"]:
+            return 1
+
+        if str(question.value).lower() in ["0", "no", "n", "false"]:
+            return 0
+
+        raise YunohostValidationError(
+            "app_argument_choice_invalid",
+            name=question.name,
+            value=question.value,
+            choices="yes, no, y, n, 1, 0",
+        )
+
+
+class DomainArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "domain"
+
+    def parse_question(self, question, user_answers):
+        from yunohost.domain import domain_list, _get_maindomain
+
+        question = super(DomainArgumentParser, self).parse_question(
+            question, user_answers
+        )
+
+        if question.default is None:
+            question.default = _get_maindomain()
+
+        question.choices = domain_list()["domains"]
+
+        return question
+
+    def _raise_invalid_answer(self, question):
+        raise YunohostValidationError(
+            "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown")
+        )
+
+
+class UserArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "user"
+
+    def parse_question(self, question, user_answers):
+        from yunohost.user import user_list, user_info
+        from yunohost.domain import _get_maindomain
+
+        question = super(UserArgumentParser, self).parse_question(
+            question, user_answers
+        )
+        question.choices = user_list()["users"]
+        if question.default is None:
+            root_mail = "root@%s" % _get_maindomain()
+            for user in question.choices.keys():
+                if root_mail in user_info(user).get("mail-aliases", []):
+                    question.default = user
+                    break
+
+        return question
+
+    def _raise_invalid_answer(self, question):
+        raise YunohostValidationError(
+            "app_argument_invalid",
+            field=question.name,
+            error=m18n.n("user_unknown", user=question.value),
+        )
+
+
+class NumberArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "number"
+    default_value = ""
+
+    def parse_question(self, question, user_answers):
+        question_parsed = super().parse_question(
+            question, user_answers
+        )
+        question_parsed.min = question.get('min', None)
+        question_parsed.max = question.get('max', None)
+        if question_parsed.default is None:
+            question_parsed.default = 0
+
+        return question_parsed
+
+    def _prevalidate(self, question):
+        super()._prevalidate(question)
+        if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()):
+            raise YunohostValidationError(
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
+            )
+
+        if question.min is not None and int(question.value) < question.min:
+            raise YunohostValidationError(
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
+            )
+
+        if question.max is not None and int(question.value) > question.max:
+            raise YunohostValidationError(
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
+            )
+
+    def _post_parse_value(self, question):
+        if isinstance(question.value, int):
+            return super()._post_parse_value(question)
+
+        if isinstance(question.value, str) and question.value.isdigit():
+            return int(question.value)
+
+        raise YunohostValidationError(
+            "app_argument_invalid", field=question.name, error=m18n.n("invalid_number")
+        )
+
+
+class DisplayTextArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "display_text"
+    readonly = True
+
+    def parse_question(self, question, user_answers):
+        question_parsed = super().parse_question(
+            question, user_answers
+        )
+
+        question_parsed.optional = True
+        question_parsed.style = question.get('style', 'info')
+
+        return question_parsed
+
+    def _format_text_for_user_input_in_cli(self, question):
+        text = question.ask['en']
+
+        if question.style in ['success', 'info', 'warning', 'danger']:
+            color = {
+                'success': 'green',
+                'info': 'cyan',
+                'warning': 'yellow',
+                'danger': 'red'
+            }
+            return colorize(m18n.g(question.style), color[question.style]) + f" {text}"
+        else:
+            return text
+
+class FileArgumentParser(YunoHostArgumentFormatParser):
+    argument_type = "file"
+    upload_dirs = []
+
+    @classmethod
+    def clean_upload_dirs(cls):
+        # Delete files uploaded from API
+        if Moulinette.interface.type== 'api':
+            for upload_dir in cls.upload_dirs:
+                if os.path.exists(upload_dir):
+                    shutil.rmtree(upload_dir)
+
+    def parse_question(self, question, user_answers):
+        question_parsed = super().parse_question(
+            question, user_answers
+        )
+        if question.get('accept'):
+            question_parsed.accept = question.get('accept').replace(' ', '').split(',')
+        else:
+            question_parsed.accept = []
+        if Moulinette.interface.type== 'api':
+            if user_answers.get(f"{question_parsed.name}[name]"):
+                question_parsed.value = {
+                    'content': question_parsed.value,
+                    'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name),
+                }
+        # If path file are the same
+        if question_parsed.value and str(question_parsed.value) == question_parsed.current_value:
+            question_parsed.value = None
+
+        return question_parsed
+
+    def _prevalidate(self, question):
+        super()._prevalidate(question)
+        if isinstance(question.value, str) and question.value and not os.path.exists(question.value):
+            raise YunohostValidationError(
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1")
+            )
+        if question.value in [None, ''] or not question.accept:
+            return
+
+        filename = question.value if isinstance(question.value, str) else question.value['filename']
+        if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept:
+            raise YunohostValidationError(
+                "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2")
+            )
+
+
+    def _post_parse_value(self, question):
+        from base64 import b64decode
+        # Upload files from API
+        # A file arg contains a string with "FILENAME:BASE64_CONTENT"
+        if not question.value:
+            return question.value
+
+        if Moulinette.interface.type== 'api':
+
+            upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_')
+            FileArgumentParser.upload_dirs += [upload_dir]
+            filename = question.value['filename']
+            logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}")
+
+            # Filename is given by user of the API. For security reason, we have replaced
+            # os.path.join to avoid the user to be able to rewrite a file in filesystem
+            # i.e. os.path.join("/foo", "/etc/passwd") == "/etc/passwd"
+            file_path = os.path.normpath(upload_dir + "/" + filename)
+            if not file_path.startswith(upload_dir + "/"):
+                raise YunohostError("relative_parent_path_in_filename_forbidden")
+            i = 2
+            while os.path.exists(file_path):
+                file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i))
+                i += 1
+            content = question.value['content']
+            try:
+                with open(file_path, 'wb') as f:
+                    f.write(b64decode(content))
+            except IOError as e:
+                raise YunohostError("cannot_write_file", file=file_path, error=str(e))
+            except Exception as e:
+                raise YunohostError("error_writing_file", file=file_path, error=str(e))
+            question.value = file_path
+        return question.value
+
+
+ARGUMENTS_TYPE_PARSERS = {
+    "string": StringArgumentParser,
+    "text": StringArgumentParser,
+    "select": StringArgumentParser,
+    "tags": TagsArgumentParser,
+    "email": StringArgumentParser,
+    "url": StringArgumentParser,
+    "date": StringArgumentParser,
+    "time": StringArgumentParser,
+    "color": StringArgumentParser,
+    "password": PasswordArgumentParser,
+    "path": PathArgumentParser,
+    "boolean": BooleanArgumentParser,
+    "domain": DomainArgumentParser,
+    "user": UserArgumentParser,
+    "number": NumberArgumentParser,
+    "range": NumberArgumentParser,
+    "display_text": DisplayTextArgumentParser,
+    "alert": DisplayTextArgumentParser,
+    "markdown": DisplayTextArgumentParser,
+    "file": FileArgumentParser,
+}
+
+def parse_args_in_yunohost_format(user_answers, argument_questions):
+    """Parse arguments store in either manifest.json or actions.json or from a
+    config panel against the user answers when they are present.
+
+    Keyword arguments:
+        user_answers -- a dictionnary of arguments from the user (generally
+                        empty in CLI, filed from the admin interface)
+        argument_questions -- the arguments description store in yunohost
+                              format from actions.json/toml, manifest.json/toml
+                              or config_panel.json/toml
+    """
+    parsed_answers_dict = OrderedDict()
+
+    for question in argument_questions:
+        parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]()
+
+        answer = parser.parse(question=question, user_answers=user_answers)
+        if answer is not None:
+            parsed_answers_dict[question["name"]] = answer
+
+    return parsed_answers_dict
+
diff --git a/src/yunohost/utils/i18n.py b/src/yunohost/utils/i18n.py
new file mode 100644
index 000000000..89d1d0b34
--- /dev/null
+++ b/src/yunohost/utils/i18n.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2018 YUNOHOST.ORG
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+from moulinette import Moulinette, m18n
+
+def _value_for_locale(values):
+    """
+    Return proper value for current locale
+
+    Keyword arguments:
+        values -- A dict of values associated to their locale
+
+    Returns:
+        An utf-8 encoded string
+
+    """
+    if not isinstance(values, dict):
+        return values
+
+    for lang in [m18n.locale, m18n.default_locale]:
+        try:
+            return values[lang]
+        except KeyError:
+            continue
+
+    # Fallback to first value
+    return list(values.values())[0]
+
+